From 75ad350228fad38d78f2073cd18d51b168d65207 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Fri, 9 Apr 2021 10:35:44 +0200 Subject: [PATCH] [RAC] Rule registry plugin (#95903) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- docs/developer/plugin-list.asciidoc | 4 + typings/elasticsearch/search.d.ts | 10 + .../apm/common/environment_filter_values.ts | 18 + x-pack/plugins/apm/kibana.json | 4 +- .../scripts/optimize-tsconfig/tsconfig.json | 1 + .../apm/server/lib/alerts/action_variables.ts | 12 +- .../server/lib/alerts/alerting_es_client.ts | 28 +- .../alerts/create_apm_lifecycle_rule_type.ts | 11 + .../server/lib/alerts/register_apm_alerts.ts | 33 +- .../register_error_count_alert_type.test.ts | 223 +- .../alerts/register_error_count_alert_type.ts | 248 +- ...egister_transaction_duration_alert_type.ts | 212 +- ...action_duration_anomaly_alert_type.test.ts | 312 +- ...transaction_duration_anomaly_alert_type.ts | 381 +- ..._transaction_error_rate_alert_type.test.ts | 303 +- ...ister_transaction_error_rate_alert_type.ts | 284 +- .../apm/server/lib/alerts/test_utils/index.ts | 64 + x-pack/plugins/apm/server/plugin.ts | 34 +- x-pack/plugins/apm/server/routes/typings.ts | 18 - x-pack/plugins/apm/server/types.ts | 2 +- x-pack/plugins/apm/tsconfig.json | 1 + .../server/es/cluster_client_adapter.test.ts | 3 +- .../server/es/cluster_client_adapter.ts | 40 +- x-pack/plugins/event_log/server/es/context.ts | 2 +- x-pack/plugins/event_log/server/index.ts | 4 + x-pack/plugins/observability/kibana.json | 2 +- x-pack/plugins/observability/server/plugin.ts | 27 +- x-pack/plugins/observability/tsconfig.json | 1 + x-pack/plugins/rule_registry/README.md | 68 + x-pack/plugins/rule_registry/common/index.ts | 8 + x-pack/plugins/rule_registry/common/types.ts | 20 + x-pack/plugins/rule_registry/jest.config.js | 12 + x-pack/plugins/rule_registry/kibana.json | 13 + .../scripts/generate_ecs_fieldmap/index.js | 81 + .../server/generated/ecs_field_map.ts | 3374 ++++++++++++++++ .../server/generated/ecs_mappings.json | 3416 +++++++++++++++++ x-pack/plugins/rule_registry/server/index.ts | 29 + x-pack/plugins/rule_registry/server/plugin.ts | 49 + .../index.ts | 174 + .../types.ts | 51 + .../rule_registry/defaults/field_map.ts | 34 + .../rule_registry/defaults/ilm_policy.ts | 28 + .../field_map/mapping_from_field_map.ts | 32 + .../field_map/merge_field_maps.ts | 50 + .../field_map/pick_with_patterns.test.ts | 71 + .../field_map/pick_with_patterns.ts | 66 + .../runtime_type_from_fieldmap.test.ts | 95 + .../field_map/runtime_type_from_fieldmap.ts | 108 + .../server/rule_registry/index.ts | 240 ++ .../create_lifecycle_rule_type_factory.ts | 235 ++ .../server/rule_registry/types.ts | 44 + x-pack/plugins/rule_registry/server/types.ts | 100 + x-pack/plugins/rule_registry/tsconfig.json | 15 + x-pack/scripts/functional_tests.js | 1 + .../test/apm_api_integration/common/config.ts | 13 +- .../test/apm_api_integration/configs/index.ts | 6 + .../test/apm_api_integration/rules/config.ts | 10 + .../tests/alerts/rule_registry.ts | 387 ++ .../test/apm_api_integration/tests/index.ts | 4 + 59 files changed, 9812 insertions(+), 1304 deletions(-) create mode 100644 x-pack/plugins/apm/server/lib/alerts/create_apm_lifecycle_rule_type.ts create mode 100644 x-pack/plugins/apm/server/lib/alerts/test_utils/index.ts create mode 100644 x-pack/plugins/rule_registry/README.md create mode 100644 x-pack/plugins/rule_registry/common/index.ts create mode 100644 x-pack/plugins/rule_registry/common/types.ts create mode 100644 x-pack/plugins/rule_registry/jest.config.js create mode 100644 x-pack/plugins/rule_registry/kibana.json create mode 100644 x-pack/plugins/rule_registry/scripts/generate_ecs_fieldmap/index.js create mode 100644 x-pack/plugins/rule_registry/server/generated/ecs_field_map.ts create mode 100644 x-pack/plugins/rule_registry/server/generated/ecs_mappings.json create mode 100644 x-pack/plugins/rule_registry/server/index.ts create mode 100644 x-pack/plugins/rule_registry/server/plugin.ts create mode 100644 x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/index.ts create mode 100644 x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/types.ts create mode 100644 x-pack/plugins/rule_registry/server/rule_registry/defaults/field_map.ts create mode 100644 x-pack/plugins/rule_registry/server/rule_registry/defaults/ilm_policy.ts create mode 100644 x-pack/plugins/rule_registry/server/rule_registry/field_map/mapping_from_field_map.ts create mode 100644 x-pack/plugins/rule_registry/server/rule_registry/field_map/merge_field_maps.ts create mode 100644 x-pack/plugins/rule_registry/server/rule_registry/field_map/pick_with_patterns.test.ts create mode 100644 x-pack/plugins/rule_registry/server/rule_registry/field_map/pick_with_patterns.ts create mode 100644 x-pack/plugins/rule_registry/server/rule_registry/field_map/runtime_type_from_fieldmap.test.ts create mode 100644 x-pack/plugins/rule_registry/server/rule_registry/field_map/runtime_type_from_fieldmap.ts create mode 100644 x-pack/plugins/rule_registry/server/rule_registry/index.ts create mode 100644 x-pack/plugins/rule_registry/server/rule_registry/rule_type_helpers/create_lifecycle_rule_type_factory.ts create mode 100644 x-pack/plugins/rule_registry/server/rule_registry/types.ts create mode 100644 x-pack/plugins/rule_registry/server/types.ts create mode 100644 x-pack/plugins/rule_registry/tsconfig.json create mode 100644 x-pack/test/apm_api_integration/rules/config.ts create mode 100644 x-pack/test/apm_api_integration/tests/alerts/rule_registry.ts diff --git a/docs/developer/plugin-list.asciidoc b/docs/developer/plugin-list.asciidoc index 1827fdb50d1d9..a38b885aefa78 100644 --- a/docs/developer/plugin-list.asciidoc +++ b/docs/developer/plugin-list.asciidoc @@ -485,6 +485,10 @@ Elastic. |Welcome to the Kibana rollup plugin! This plugin provides Kibana support for Elasticsearch's rollup feature. Please refer to the Elasticsearch documentation to understand rollup indices and how to create rollup jobs. +|{kib-repo}blob/{branch}/x-pack/plugins/rule_registry/README.md[ruleRegistry] +|The rule registry plugin aims to make it easy for rule type producers to have their rules produce the data that they need to build rich experiences on top of a unified experience, without the risk of mapping conflicts. + + |{kib-repo}blob/{branch}/x-pack/plugins/runtime_fields/README.md[runtimeFields] |Welcome to the home of the runtime field editor and everything related to runtime fields! diff --git a/typings/elasticsearch/search.d.ts b/typings/elasticsearch/search.d.ts index fce08df1c0a04..c9bf3b1d8b7bc 100644 --- a/typings/elasticsearch/search.d.ts +++ b/typings/elasticsearch/search.d.ts @@ -370,6 +370,16 @@ export type AggregateOf< missing: { doc_count: number; } & SubAggregateOf; + multi_terms: { + doc_count_error_upper_bound: number; + sum_other_doc_count: number; + buckets: Array< + { + doc_count: number; + key: string[]; + } & SubAggregateOf + >; + }; nested: { doc_count: number; } & SubAggregateOf; diff --git a/x-pack/plugins/apm/common/environment_filter_values.ts b/x-pack/plugins/apm/common/environment_filter_values.ts index e091b53b2e5b8..c80541ee1ba6b 100644 --- a/x-pack/plugins/apm/common/environment_filter_values.ts +++ b/x-pack/plugins/apm/common/environment_filter_values.ts @@ -22,11 +22,13 @@ const environmentLabels: Record = { }; export const ENVIRONMENT_ALL = { + esFieldValue: undefined, value: ENVIRONMENT_ALL_VALUE, text: environmentLabels[ENVIRONMENT_ALL_VALUE], }; export const ENVIRONMENT_NOT_DEFINED = { + esFieldValue: undefined, value: ENVIRONMENT_NOT_DEFINED_VALUE, text: environmentLabels[ENVIRONMENT_NOT_DEFINED_VALUE], }; @@ -35,6 +37,22 @@ export function getEnvironmentLabel(environment: string) { return environmentLabels[environment] || environment; } +export function parseEnvironmentUrlParam(environment: string) { + if (environment === ENVIRONMENT_ALL_VALUE) { + return ENVIRONMENT_ALL; + } + + if (environment === ENVIRONMENT_NOT_DEFINED_VALUE) { + return ENVIRONMENT_NOT_DEFINED; + } + + return { + esFieldValue: environment, + value: environment, + text: environment, + }; +} + // returns the environment url param that should be used // based on the requested environment. If the requested // environment is different from the URL parameter, we'll diff --git a/x-pack/plugins/apm/kibana.json b/x-pack/plugins/apm/kibana.json index e340f8bf19126..28e4a7b36e740 100644 --- a/x-pack/plugins/apm/kibana.json +++ b/x-pack/plugins/apm/kibana.json @@ -9,7 +9,8 @@ "licensing", "triggersActionsUi", "embeddable", - "infra" + "infra", + "observability" ], "optionalPlugins": [ "spaces", @@ -18,7 +19,6 @@ "taskManager", "actions", "alerting", - "observability", "security", "ml", "home", diff --git a/x-pack/plugins/apm/scripts/optimize-tsconfig/tsconfig.json b/x-pack/plugins/apm/scripts/optimize-tsconfig/tsconfig.json index 319eb53313231..40d42298b967b 100644 --- a/x-pack/plugins/apm/scripts/optimize-tsconfig/tsconfig.json +++ b/x-pack/plugins/apm/scripts/optimize-tsconfig/tsconfig.json @@ -2,6 +2,7 @@ "include": [ "./x-pack/plugins/apm/**/*", "./x-pack/plugins/observability/**/*", + "./x-pack/plugins/rule_registry/**/*", "./typings/**/*" ], "exclude": [ diff --git a/x-pack/plugins/apm/server/lib/alerts/action_variables.ts b/x-pack/plugins/apm/server/lib/alerts/action_variables.ts index 473912c4177a9..b065da7123dec 100644 --- a/x-pack/plugins/apm/server/lib/alerts/action_variables.ts +++ b/x-pack/plugins/apm/server/lib/alerts/action_variables.ts @@ -13,28 +13,28 @@ export const apmActionVariables = { 'xpack.apm.alerts.action_variables.serviceName', { defaultMessage: 'The service the alert is created for' } ), - name: 'serviceName', + name: 'serviceName' as const, }, transactionType: { description: i18n.translate( 'xpack.apm.alerts.action_variables.transactionType', { defaultMessage: 'The transaction type the alert is created for' } ), - name: 'transactionType', + name: 'transactionType' as const, }, environment: { description: i18n.translate( 'xpack.apm.alerts.action_variables.environment', { defaultMessage: 'The transaction type the alert is created for' } ), - name: 'environment', + name: 'environment' as const, }, threshold: { description: i18n.translate('xpack.apm.alerts.action_variables.threshold', { defaultMessage: 'Any trigger value above this value will cause the alert to fire', }), - name: 'threshold', + name: 'threshold' as const, }, triggerValue: { description: i18n.translate( @@ -44,7 +44,7 @@ export const apmActionVariables = { 'The value that breached the threshold and triggered the alert', } ), - name: 'triggerValue', + name: 'triggerValue' as const, }, interval: { description: i18n.translate( @@ -54,6 +54,6 @@ export const apmActionVariables = { 'The length and unit of the time period where the alert conditions were met', } ), - name: 'interval', + name: 'interval' as const, }, }; diff --git a/x-pack/plugins/apm/server/lib/alerts/alerting_es_client.ts b/x-pack/plugins/apm/server/lib/alerts/alerting_es_client.ts index 9a0ba514bb479..e3d5e5481caa5 100644 --- a/x-pack/plugins/apm/server/lib/alerts/alerting_es_client.ts +++ b/x-pack/plugins/apm/server/lib/alerts/alerting_es_client.ts @@ -5,28 +5,24 @@ * 2.0. */ -import { ApiResponse } from '@elastic/elasticsearch'; -import { ThresholdMetActionGroupId } from '../../../common/alert_types'; import { ESSearchRequest, ESSearchResponse, } from '../../../../../../typings/elasticsearch'; -import { - AlertInstanceContext, - AlertInstanceState, - AlertServices, -} from '../../../../alerting/server'; +import { AlertServices } from '../../../../alerting/server'; -export function alertingEsClient( - services: AlertServices< - AlertInstanceState, - AlertInstanceContext, - ThresholdMetActionGroupId - >, +export async function alertingEsClient( + scopedClusterClient: AlertServices< + never, + never, + never + >['scopedClusterClient'], params: TParams -): Promise>> { - return (services.scopedClusterClient.asCurrentUser.search({ +): Promise> { + const response = await scopedClusterClient.asCurrentUser.search({ ...params, ignore_unavailable: true, - }) as unknown) as Promise>>; + }); + + return (response.body as unknown) as ESSearchResponse; } diff --git a/x-pack/plugins/apm/server/lib/alerts/create_apm_lifecycle_rule_type.ts b/x-pack/plugins/apm/server/lib/alerts/create_apm_lifecycle_rule_type.ts new file mode 100644 index 0000000000000..8d250a5765cce --- /dev/null +++ b/x-pack/plugins/apm/server/lib/alerts/create_apm_lifecycle_rule_type.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createLifecycleRuleTypeFactory } from '../../../../rule_registry/server'; +import { APMRuleRegistry } from '../../plugin'; + +export const createAPMLifecycleRuleType = createLifecycleRuleTypeFactory(); diff --git a/x-pack/plugins/apm/server/lib/alerts/register_apm_alerts.ts b/x-pack/plugins/apm/server/lib/alerts/register_apm_alerts.ts index a9824c130faa5..9a362efa90ac0 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_apm_alerts.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_apm_alerts.ts @@ -6,38 +6,25 @@ */ import { Observable } from 'rxjs'; -import { AlertingPlugin } from '../../../../alerting/server'; -import { ActionsPlugin } from '../../../../actions/server'; +import { Logger } from 'kibana/server'; import { registerTransactionDurationAlertType } from './register_transaction_duration_alert_type'; import { registerTransactionDurationAnomalyAlertType } from './register_transaction_duration_anomaly_alert_type'; import { registerErrorCountAlertType } from './register_error_count_alert_type'; import { APMConfig } from '../..'; import { MlPluginSetup } from '../../../../ml/server'; import { registerTransactionErrorRateAlertType } from './register_transaction_error_rate_alert_type'; +import { APMRuleRegistry } from '../../plugin'; -interface Params { - alerting: AlertingPlugin['setup']; - actions: ActionsPlugin['setup']; +export interface RegisterRuleDependencies { + registry: APMRuleRegistry; ml?: MlPluginSetup; config$: Observable; + logger: Logger; } -export function registerApmAlerts(params: Params) { - registerTransactionDurationAlertType({ - alerting: params.alerting, - config$: params.config$, - }); - registerTransactionDurationAnomalyAlertType({ - alerting: params.alerting, - ml: params.ml, - config$: params.config$, - }); - registerErrorCountAlertType({ - alerting: params.alerting, - config$: params.config$, - }); - registerTransactionErrorRateAlertType({ - alerting: params.alerting, - config$: params.config$, - }); +export function registerApmAlerts(dependencies: RegisterRuleDependencies) { + registerTransactionDurationAlertType(dependencies); + registerTransactionDurationAnomalyAlertType(dependencies); + registerErrorCountAlertType(dependencies); + registerTransactionErrorRateAlertType(dependencies); } diff --git a/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.test.ts b/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.test.ts index d7dd7aee3ca25..5758dea1860b2 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.test.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.test.ts @@ -5,50 +5,17 @@ * 2.0. */ -import { Observable } from 'rxjs'; -import * as Rx from 'rxjs'; -import { toArray, map } from 'rxjs/operators'; - -import { AlertingPlugin } from '../../../../alerting/server'; -import { APMConfig } from '../..'; - import { registerErrorCountAlertType } from './register_error_count_alert_type'; -import { elasticsearchServiceMock } from 'src/core/server/mocks'; // eslint-disable-next-line @kbn/eslint/no-restricted-paths import { elasticsearchClientMock } from 'src/core/server/elasticsearch/client/mocks'; - -type Operator = (source: Rx.Observable) => Rx.Observable; -const pipeClosure = (fn: Operator): Operator => { - return (source: Rx.Observable) => { - return Rx.defer(() => fn(source)); - }; -}; -const mockedConfig$ = (Rx.of('apm_oss.errorIndices').pipe( - pipeClosure((source$) => { - return source$.pipe(map((i) => i)); - }), - toArray() -) as unknown) as Observable; +import { createRuleTypeMocks } from './test_utils'; describe('Error count alert', () => { it("doesn't send an alert when error count is less than threshold", async () => { - let alertExecutor: any; - const alerting = { - registerType: ({ executor }) => { - alertExecutor = executor; - }, - } as AlertingPlugin['setup']; + const { services, dependencies, executor } = createRuleTypeMocks(); - registerErrorCountAlertType({ - alerting, - config$: mockedConfig$, - }); - expect(alertExecutor).toBeDefined(); + registerErrorCountAlertType(dependencies); - const services = { - scopedClusterClient: elasticsearchServiceMock.createScopedClusterClient(), - alertInstanceFactory: jest.fn(), - }; const params = { threshold: 1 }; services.scopedClusterClient.asCurrentUser.search.mockReturnValue( @@ -71,30 +38,21 @@ describe('Error count alert', () => { }) ); - await alertExecutor!({ services, params }); + await executor({ params }); expect(services.alertInstanceFactory).not.toBeCalled(); }); - it('sends alerts with service name and environment', async () => { - let alertExecutor: any; - const alerting = { - registerType: ({ executor }) => { - alertExecutor = executor; - }, - } as AlertingPlugin['setup']; + it('sends alerts with service name and environment for those that exceeded the threshold', async () => { + const { + services, + dependencies, + executor, + scheduleActions, + } = createRuleTypeMocks(); - registerErrorCountAlertType({ - alerting, - config$: mockedConfig$, - }); - expect(alertExecutor).toBeDefined(); + registerErrorCountAlertType(dependencies); - const scheduleActions = jest.fn(); - const services = { - scopedClusterClient: elasticsearchServiceMock.createScopedClusterClient(), - alertInstanceFactory: jest.fn(() => ({ scheduleActions })), - }; - const params = { threshold: 1, windowSize: 5, windowUnit: 'm' }; + const params = { threshold: 2, windowSize: 5, windowUnit: 'm' }; services.scopedClusterClient.asCurrentUser.search.mockReturnValue( elasticsearchClientMock.createSuccessTransportRequestPromise({ @@ -106,18 +64,62 @@ describe('Error count alert', () => { }, }, aggregations: { - services: { + error_counts: { buckets: [ { - key: 'foo', - environments: { - buckets: [{ key: 'env-foo' }, { key: 'env-foo-2' }], + key: ['foo', 'env-foo'], + doc_count: 5, + latest: { + top: [ + { + metrics: { + 'service.name': 'foo', + 'service.environment': 'env-foo', + }, + }, + ], + }, + }, + { + key: ['foo', 'env-foo-2'], + doc_count: 4, + latest: { + top: [ + { + metrics: { + 'service.name': 'foo', + 'service.environment': 'env-foo-2', + }, + }, + ], }, }, { - key: 'bar', - environments: { - buckets: [{ key: 'env-bar' }, { key: 'env-bar-2' }], + key: ['bar', 'env-bar'], + doc_count: 3, + latest: { + top: [ + { + metrics: { + 'service.name': 'bar', + 'service.environment': 'env-bar', + }, + }, + ], + }, + }, + { + key: ['bar', 'env-bar-2'], + doc_count: 1, + latest: { + top: [ + { + metrics: { + 'service.name': 'bar', + 'service.environment': 'env-bar-2', + }, + }, + ], }, }, ], @@ -134,115 +136,36 @@ describe('Error count alert', () => { }) ); - await alertExecutor!({ services, params }); + await executor({ params }); [ 'apm.error_rate_foo_env-foo', 'apm.error_rate_foo_env-foo-2', 'apm.error_rate_bar_env-bar', - 'apm.error_rate_bar_env-bar-2', ].forEach((instanceName) => expect(services.alertInstanceFactory).toHaveBeenCalledWith(instanceName) ); + expect(scheduleActions).toHaveBeenCalledTimes(3); + expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { serviceName: 'foo', environment: 'env-foo', - threshold: 1, - triggerValue: 2, + threshold: 2, + triggerValue: 5, interval: '5m', }); expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { serviceName: 'foo', environment: 'env-foo-2', - threshold: 1, - triggerValue: 2, + threshold: 2, + triggerValue: 4, interval: '5m', }); expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { serviceName: 'bar', environment: 'env-bar', - threshold: 1, - triggerValue: 2, - interval: '5m', - }); - expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { - serviceName: 'bar', - environment: 'env-bar-2', - threshold: 1, - triggerValue: 2, - interval: '5m', - }); - }); - it('sends alerts with service name', async () => { - let alertExecutor: any; - const alerting = { - registerType: ({ executor }) => { - alertExecutor = executor; - }, - } as AlertingPlugin['setup']; - - registerErrorCountAlertType({ - alerting, - config$: mockedConfig$, - }); - expect(alertExecutor).toBeDefined(); - - const scheduleActions = jest.fn(); - const services = { - scopedClusterClient: elasticsearchServiceMock.createScopedClusterClient(), - alertInstanceFactory: jest.fn(() => ({ scheduleActions })), - }; - const params = { threshold: 1, windowSize: 5, windowUnit: 'm' }; - - services.scopedClusterClient.asCurrentUser.search.mockReturnValue( - elasticsearchClientMock.createSuccessTransportRequestPromise({ - hits: { - hits: [], - total: { - relation: 'eq', - value: 2, - }, - }, - aggregations: { - services: { - buckets: [ - { - key: 'foo', - }, - { - key: 'bar', - }, - ], - }, - }, - took: 0, - timed_out: false, - _shards: { - failed: 0, - skipped: 0, - successful: 1, - total: 1, - }, - }) - ); - - await alertExecutor!({ services, params }); - ['apm.error_rate_foo', 'apm.error_rate_bar'].forEach((instanceName) => - expect(services.alertInstanceFactory).toHaveBeenCalledWith(instanceName) - ); - - expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { - serviceName: 'foo', - environment: undefined, - threshold: 1, - triggerValue: 2, - interval: '5m', - }); - expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { - serviceName: 'bar', - environment: undefined, - threshold: 1, - triggerValue: 2, + threshold: 2, + triggerValue: 3, interval: '5m', }); }); diff --git a/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.ts b/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.ts index 0120891a8f868..8240e0c369d1f 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.ts @@ -5,22 +5,11 @@ * 2.0. */ -import { schema, TypeOf } from '@kbn/config-schema'; -import { isEmpty } from 'lodash'; -import { Observable } from 'rxjs'; +import { schema } from '@kbn/config-schema'; import { take } from 'rxjs/operators'; -import { APMConfig } from '../..'; -import { - AlertingPlugin, - AlertInstanceContext, - AlertInstanceState, - AlertTypeState, -} from '../../../../alerting/server'; -import { - AlertType, - ALERT_TYPES_CONFIG, - ThresholdMetActionGroupId, -} from '../../../common/alert_types'; +import { ENVIRONMENT_NOT_DEFINED } from '../../../common/environment_filter_values'; +import { asMutableArray } from '../../../common/utils/as_mutable_array'; +import { AlertType, ALERT_TYPES_CONFIG } from '../../../common/alert_types'; import { PROCESSOR_EVENT, SERVICE_ENVIRONMENT, @@ -31,11 +20,8 @@ import { environmentQuery } from '../../../server/utils/queries'; import { getApmIndices } from '../settings/apm_indices/get_apm_indices'; import { apmActionVariables } from './action_variables'; import { alertingEsClient } from './alerting_es_client'; - -interface RegisterAlertParams { - alerting: AlertingPlugin['setup']; - config$: Observable; -} +import { RegisterRuleDependencies } from './register_apm_alerts'; +import { createAPMLifecycleRuleType } from './create_apm_lifecycle_rule_type'; const paramsSchema = schema.object({ windowSize: schema.number(), @@ -48,127 +34,131 @@ const paramsSchema = schema.object({ const alertTypeConfig = ALERT_TYPES_CONFIG[AlertType.ErrorCount]; export function registerErrorCountAlertType({ - alerting, + registry, config$, -}: RegisterAlertParams) { - alerting.registerType< - TypeOf, - AlertTypeState, - AlertInstanceState, - AlertInstanceContext, - ThresholdMetActionGroupId - >({ - id: AlertType.ErrorCount, - name: alertTypeConfig.name, - actionGroups: alertTypeConfig.actionGroups, - defaultActionGroupId: alertTypeConfig.defaultActionGroupId, - validate: { - params: paramsSchema, - }, - actionVariables: { - context: [ - apmActionVariables.serviceName, - apmActionVariables.environment, - apmActionVariables.threshold, - apmActionVariables.triggerValue, - apmActionVariables.interval, - ], - }, - producer: 'apm', - minimumLicenseRequired: 'basic', - executor: async ({ services, params }) => { - const config = await config$.pipe(take(1)).toPromise(); - const alertParams = params; - const indices = await getApmIndices({ - config, - savedObjectsClient: services.savedObjectsClient, - }); - const maxServiceEnvironments = config['xpack.apm.maxServiceEnvironments']; +}: RegisterRuleDependencies) { + registry.registerType( + createAPMLifecycleRuleType({ + id: AlertType.ErrorCount, + name: alertTypeConfig.name, + actionGroups: alertTypeConfig.actionGroups, + defaultActionGroupId: alertTypeConfig.defaultActionGroupId, + validate: { + params: paramsSchema, + }, + actionVariables: { + context: [ + apmActionVariables.serviceName, + apmActionVariables.environment, + apmActionVariables.threshold, + apmActionVariables.triggerValue, + apmActionVariables.interval, + ], + }, + producer: 'apm', + minimumLicenseRequired: 'basic', + executor: async ({ services, params }) => { + const config = await config$.pipe(take(1)).toPromise(); + const alertParams = params; + const indices = await getApmIndices({ + config, + savedObjectsClient: services.savedObjectsClient, + }); - const searchParams = { - index: indices['apm_oss.errorIndices'], - size: 0, - body: { - track_total_hits: true, - query: { - bool: { - filter: [ - { - range: { - '@timestamp': { - gte: `now-${alertParams.windowSize}${alertParams.windowUnit}`, + const searchParams = { + index: indices['apm_oss.errorIndices'], + size: 0, + body: { + query: { + bool: { + filter: [ + { + range: { + '@timestamp': { + gte: `now-${alertParams.windowSize}${alertParams.windowUnit}`, + }, }, }, - }, - { term: { [PROCESSOR_EVENT]: ProcessorEvent.error } }, - ...(alertParams.serviceName - ? [{ term: { [SERVICE_NAME]: alertParams.serviceName } }] - : []), - ...environmentQuery(alertParams.environment), - ], - }, - }, - aggs: { - services: { - terms: { - field: SERVICE_NAME, - size: 50, + { term: { [PROCESSOR_EVENT]: ProcessorEvent.error } }, + ...(alertParams.serviceName + ? [{ term: { [SERVICE_NAME]: alertParams.serviceName } }] + : []), + ...environmentQuery(alertParams.environment), + ], }, - aggs: { - environments: { - terms: { - field: SERVICE_ENVIRONMENT, - size: maxServiceEnvironments, + }, + aggs: { + error_counts: { + multi_terms: { + terms: [ + { field: SERVICE_NAME }, + { field: SERVICE_ENVIRONMENT, missing: '' }, + ], + size: 10000, + }, + aggs: { + latest: { + top_metrics: { + metrics: asMutableArray([ + { field: SERVICE_NAME }, + { field: SERVICE_ENVIRONMENT }, + ] as const), + sort: { + '@timestamp': 'desc' as const, + }, + }, }, }, }, }, }, - }, - }; + }; + + const response = await alertingEsClient( + services.scopedClusterClient, + searchParams + ); - const { body: response } = await alertingEsClient(services, searchParams); - const errorCount = response.hits.total.value; + const errorCountResults = + response.aggregations?.error_counts.buckets.map((bucket) => { + const latest = bucket.latest.top[0].metrics; - if (errorCount > alertParams.threshold) { - function scheduleAction({ - serviceName, - environment, - }: { - serviceName: string; - environment?: string; - }) { - const alertInstanceName = [ - AlertType.ErrorCount, - serviceName, - environment, - ] - .filter((name) => name) - .join('_'); + return { + serviceName: latest['service.name'] as string, + environment: latest['service.environment'] as string | undefined, + errorCount: bucket.doc_count, + }; + }) ?? []; - const alertInstance = services.alertInstanceFactory( - alertInstanceName - ); - alertInstance.scheduleActions(alertTypeConfig.defaultActionGroupId, { - serviceName, - environment, - threshold: alertParams.threshold, - triggerValue: errorCount, - interval: `${alertParams.windowSize}${alertParams.windowUnit}`, + errorCountResults + .filter((result) => result.errorCount >= alertParams.threshold) + .forEach((result) => { + const { serviceName, environment, errorCount } = result; + + services + .alertWithLifecycle({ + id: [AlertType.ErrorCount, serviceName, environment] + .filter((name) => name) + .join('_'), + fields: { + [SERVICE_NAME]: serviceName, + ...(environment + ? { [SERVICE_ENVIRONMENT]: environment } + : {}), + [PROCESSOR_EVENT]: 'error', + }, + }) + .scheduleActions(alertTypeConfig.defaultActionGroupId, { + serviceName, + environment: environment || ENVIRONMENT_NOT_DEFINED.text, + threshold: alertParams.threshold, + triggerValue: errorCount, + interval: `${alertParams.windowSize}${alertParams.windowUnit}`, + }); }); - } - response.aggregations?.services.buckets.forEach((serviceBucket) => { - const serviceName = serviceBucket.key as string; - if (isEmpty(serviceBucket.environments?.buckets)) { - scheduleAction({ serviceName }); - } else { - serviceBucket.environments.buckets.forEach((envBucket) => { - const environment = envBucket.key as string; - scheduleAction({ serviceName, environment }); - }); - } - }); - } - }, - }); + + return {}; + }, + }) + ); } diff --git a/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_alert_type.ts b/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_alert_type.ts index 500e0744d5638..6ca1c4370d6ae 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_alert_type.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_alert_type.ts @@ -6,10 +6,9 @@ */ import { schema } from '@kbn/config-schema'; -import { Observable } from 'rxjs'; import { take } from 'rxjs/operators'; -import { APMConfig } from '../..'; -import { AlertingPlugin } from '../../../../alerting/server'; +import { QueryContainer } from '@elastic/elasticsearch/api/types'; +import { parseEnvironmentUrlParam } from '../../../common/environment_filter_values'; import { AlertType, ALERT_TYPES_CONFIG } from '../../../common/alert_types'; import { PROCESSOR_EVENT, @@ -24,11 +23,8 @@ import { environmentQuery } from '../../../server/utils/queries'; import { getApmIndices } from '../settings/apm_indices/get_apm_indices'; import { apmActionVariables } from './action_variables'; import { alertingEsClient } from './alerting_es_client'; - -interface RegisterAlertParams { - alerting: AlertingPlugin['setup']; - config$: Observable; -} +import { RegisterRuleDependencies } from './register_apm_alerts'; +import { createAPMLifecycleRuleType } from './create_apm_lifecycle_rule_type'; const paramsSchema = schema.object({ serviceName: schema.string(), @@ -47,116 +43,126 @@ const paramsSchema = schema.object({ const alertTypeConfig = ALERT_TYPES_CONFIG[AlertType.TransactionDuration]; export function registerTransactionDurationAlertType({ - alerting, + registry, config$, -}: RegisterAlertParams) { - alerting.registerType({ - id: AlertType.TransactionDuration, - name: alertTypeConfig.name, - actionGroups: alertTypeConfig.actionGroups, - defaultActionGroupId: alertTypeConfig.defaultActionGroupId, - validate: { - params: paramsSchema, - }, - actionVariables: { - context: [ - apmActionVariables.serviceName, - apmActionVariables.transactionType, - apmActionVariables.environment, - apmActionVariables.threshold, - apmActionVariables.triggerValue, - apmActionVariables.interval, - ], - }, - producer: 'apm', - minimumLicenseRequired: 'basic', - executor: async ({ services, params }) => { - const config = await config$.pipe(take(1)).toPromise(); - const alertParams = params; - const indices = await getApmIndices({ - config, - savedObjectsClient: services.savedObjectsClient, - }); - const maxServiceEnvironments = config['xpack.apm.maxServiceEnvironments']; +}: RegisterRuleDependencies) { + registry.registerType( + createAPMLifecycleRuleType({ + id: AlertType.TransactionDuration, + name: alertTypeConfig.name, + actionGroups: alertTypeConfig.actionGroups, + defaultActionGroupId: alertTypeConfig.defaultActionGroupId, + validate: { + params: paramsSchema, + }, + actionVariables: { + context: [ + apmActionVariables.serviceName, + apmActionVariables.transactionType, + apmActionVariables.environment, + apmActionVariables.threshold, + apmActionVariables.triggerValue, + apmActionVariables.interval, + ], + }, + producer: 'apm', + minimumLicenseRequired: 'basic', + executor: async ({ services, params }) => { + const config = await config$.pipe(take(1)).toPromise(); + const alertParams = params; + const indices = await getApmIndices({ + config, + savedObjectsClient: services.savedObjectsClient, + }); - const searchParams = { - index: indices['apm_oss.transactionIndices'], - size: 0, - body: { - query: { - bool: { - filter: [ - { - range: { - '@timestamp': { - gte: `now-${alertParams.windowSize}${alertParams.windowUnit}`, + const searchParams = { + index: indices['apm_oss.transactionIndices'], + size: 0, + body: { + query: { + bool: { + filter: [ + { + range: { + '@timestamp': { + gte: `now-${alertParams.windowSize}${alertParams.windowUnit}`, + }, }, }, - }, - { term: { [PROCESSOR_EVENT]: ProcessorEvent.transaction } }, - { term: { [SERVICE_NAME]: alertParams.serviceName } }, - { term: { [TRANSACTION_TYPE]: alertParams.transactionType } }, - ...environmentQuery(alertParams.environment), - ], + { term: { [PROCESSOR_EVENT]: ProcessorEvent.transaction } }, + { term: { [SERVICE_NAME]: alertParams.serviceName } }, + { term: { [TRANSACTION_TYPE]: alertParams.transactionType } }, + ...environmentQuery(alertParams.environment), + ] as QueryContainer[], + }, }, - }, - aggs: { - agg: - alertParams.aggregationType === 'avg' - ? { avg: { field: TRANSACTION_DURATION } } - : { - percentiles: { - field: TRANSACTION_DURATION, - percents: [ - alertParams.aggregationType === '95th' ? 95 : 99, - ], + aggs: { + latency: + alertParams.aggregationType === 'avg' + ? { avg: { field: TRANSACTION_DURATION } } + : { + percentiles: { + field: TRANSACTION_DURATION, + percents: [ + alertParams.aggregationType === '95th' ? 95 : 99, + ], + }, }, - }, - environments: { - terms: { - field: SERVICE_ENVIRONMENT, - size: maxServiceEnvironments, - }, }, }, - }, - }; + }; - const { body: response } = await alertingEsClient(services, searchParams); + const response = await alertingEsClient( + services.scopedClusterClient, + searchParams + ); - if (!response.aggregations) { - return; - } + if (!response.aggregations) { + return {}; + } - const { agg, environments } = response.aggregations; + const { latency } = response.aggregations; - const transactionDuration = - 'values' in agg ? Object.values(agg.values)[0] : agg?.value; + const transactionDuration = + 'values' in latency + ? Object.values(latency.values)[0] + : latency?.value; - const threshold = alertParams.threshold * 1000; + const threshold = alertParams.threshold * 1000; - if (transactionDuration && transactionDuration > threshold) { - const durationFormatter = getDurationFormatter(transactionDuration); - const transactionDurationFormatted = durationFormatter( - transactionDuration - ).formatted; + if (transactionDuration && transactionDuration > threshold) { + const durationFormatter = getDurationFormatter(transactionDuration); + const transactionDurationFormatted = durationFormatter( + transactionDuration + ).formatted; - environments.buckets.map((bucket) => { - const environment = bucket.key; - const alertInstance = services.alertInstanceFactory( - `${AlertType.TransactionDuration}_${environment}` + const environmentParsed = parseEnvironmentUrlParam( + alertParams.environment ); - alertInstance.scheduleActions(alertTypeConfig.defaultActionGroupId, { - transactionType: alertParams.transactionType, - serviceName: alertParams.serviceName, - environment, - threshold, - triggerValue: transactionDurationFormatted, - interval: `${alertParams.windowSize}${alertParams.windowUnit}`, - }); - }); - } - }, - }); + services + .alertWithLifecycle({ + id: `${AlertType.TransactionDuration}_${environmentParsed.text}`, + fields: { + [SERVICE_NAME]: alertParams.serviceName, + ...(environmentParsed.esFieldValue + ? { [SERVICE_ENVIRONMENT]: environmentParsed.esFieldValue } + : {}), + [TRANSACTION_TYPE]: alertParams.transactionType, + }, + }) + .scheduleActions(alertTypeConfig.defaultActionGroupId, { + transactionType: alertParams.transactionType, + serviceName: alertParams.serviceName, + environment: environmentParsed.text, + threshold, + triggerValue: transactionDurationFormatted, + interval: `${alertParams.windowSize}${alertParams.windowUnit}`, + }); + } + + return {}; + }, + }) + ); } diff --git a/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.test.ts b/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.test.ts index 5f6c07cae4b8f..b9346b2bf4649 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.test.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.test.ts @@ -4,29 +4,11 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - -import { Observable } from 'rxjs'; -import * as Rx from 'rxjs'; -import { toArray, map } from 'rxjs/operators'; -import { AlertingPlugin } from '../../../../alerting/server'; import { registerTransactionDurationAnomalyAlertType } from './register_transaction_duration_anomaly_alert_type'; -import { APMConfig } from '../..'; import { ANOMALY_SEVERITY } from '../../../../ml/common'; import { Job, MlPluginSetup } from '../../../../ml/server'; import * as GetServiceAnomalies from '../service_map/get_service_anomalies'; - -type Operator = (source: Rx.Observable) => Rx.Observable; -const pipeClosure = (fn: Operator): Operator => { - return (source: Rx.Observable) => { - return Rx.defer(() => fn(source)); - }; -}; -const mockedConfig$ = (Rx.of('apm_oss.errorIndices').pipe( - pipeClosure((source$) => { - return source$.pipe(map((i) => i)); - }), - toArray() -) as unknown) as Observable; +import { createRuleTypeMocks } from './test_utils'; describe('Transaction duration anomaly alert', () => { afterEach(() => { @@ -34,28 +16,21 @@ describe('Transaction duration anomaly alert', () => { }); describe("doesn't send alert", () => { it('ml is not defined', async () => { - let alertExecutor: any; - const alerting = { - registerType: ({ executor }) => { - alertExecutor = executor; - }, - } as AlertingPlugin['setup']; + const { services, dependencies, executor } = createRuleTypeMocks(); registerTransactionDurationAnomalyAlertType({ - alerting, + ...dependencies, ml: undefined, - config$: mockedConfig$, }); - expect(alertExecutor).toBeDefined(); - const services = { - callCluster: jest.fn(), - alertInstanceFactory: jest.fn(), - }; const params = { anomalySeverityType: ANOMALY_SEVERITY.MINOR }; - await alertExecutor!({ services, params }); - expect(services.callCluster).not.toHaveBeenCalled(); + await executor({ params }); + + expect( + services.scopedClusterClient.asCurrentUser.search + ).not.toHaveBeenCalled(); + expect(services.alertInstanceFactory).not.toHaveBeenCalled(); }); @@ -64,13 +39,7 @@ describe('Transaction duration anomaly alert', () => { .spyOn(GetServiceAnomalies, 'getMLJobs') .mockReturnValue(Promise.resolve([])); - let alertExecutor: any; - - const alerting = { - registerType: ({ executor }) => { - alertExecutor = executor; - }, - } as AlertingPlugin['setup']; + const { services, dependencies, executor } = createRuleTypeMocks(); const ml = ({ mlSystemProvider: () => ({ mlAnomalySearch: jest.fn() }), @@ -78,117 +47,47 @@ describe('Transaction duration anomaly alert', () => { } as unknown) as MlPluginSetup; registerTransactionDurationAnomalyAlertType({ - alerting, + ...dependencies, ml, - config$: mockedConfig$, }); - expect(alertExecutor).toBeDefined(); - const services = { - callCluster: jest.fn(), - alertInstanceFactory: jest.fn(), - }; const params = { anomalySeverityType: ANOMALY_SEVERITY.MINOR }; - await alertExecutor!({ services, params }); - expect(services.callCluster).not.toHaveBeenCalled(); - expect(services.alertInstanceFactory).not.toHaveBeenCalled(); - }); + await executor({ params }); + expect( + services.scopedClusterClient.asCurrentUser.search + ).not.toHaveBeenCalled(); - it('anomaly is less than threshold', async () => { - jest - .spyOn(GetServiceAnomalies, 'getMLJobs') - .mockReturnValue( - Promise.resolve([{ job_id: '1' }, { job_id: '2' }] as Job[]) - ); - - let alertExecutor: any; - - const alerting = { - registerType: ({ executor }) => { - alertExecutor = executor; - }, - } as AlertingPlugin['setup']; - - const ml = ({ - mlSystemProvider: () => ({ - mlAnomalySearch: () => ({ - hits: { total: { value: 0 } }, - }), - }), - anomalyDetectorsProvider: jest.fn(), - } as unknown) as MlPluginSetup; - - registerTransactionDurationAnomalyAlertType({ - alerting, - ml, - config$: mockedConfig$, - }); - expect(alertExecutor).toBeDefined(); - - const services = { - callCluster: jest.fn(), - alertInstanceFactory: jest.fn(), - }; - const params = { anomalySeverityType: ANOMALY_SEVERITY.MINOR }; - - await alertExecutor!({ services, params }); - expect(services.callCluster).not.toHaveBeenCalled(); expect(services.alertInstanceFactory).not.toHaveBeenCalled(); }); - }); - describe('sends alert', () => { - it('with service name, environment and transaction type', async () => { + it('anomaly is less than threshold', async () => { jest.spyOn(GetServiceAnomalies, 'getMLJobs').mockReturnValue( - Promise.resolve([ + Promise.resolve(([ { job_id: '1', - custom_settings: { - job_tags: { - environment: 'production', - }, - }, - } as unknown, + custom_settings: { job_tags: { environment: 'development' } }, + }, { job_id: '2', - custom_settings: { - job_tags: { - environment: 'production', - }, - }, - } as unknown, - ] as Job[]) + custom_settings: { job_tags: { environment: 'production' } }, + }, + ] as unknown) as Job[]) ); - let alertExecutor: any; - - const alerting = { - registerType: ({ executor }) => { - alertExecutor = executor; - }, - } as AlertingPlugin['setup']; + const { services, dependencies, executor } = createRuleTypeMocks(); const ml = ({ mlSystemProvider: () => ({ mlAnomalySearch: () => ({ - hits: { total: { value: 2 } }, aggregations: { - services: { + anomaly_groups: { buckets: [ { - key: 'foo', - transaction_types: { - buckets: [{ key: 'type-foo' }], - }, - record_avg: { value: 80 }, - }, - { - key: 'bar', - transaction_types: { - buckets: [{ key: 'type-bar' }], + doc_count: 1, + latest_score: { + top: [{ metrics: { record_score: 0, job_id: '1' } }], }, - record_avg: { value: 20 }, }, ], }, @@ -199,84 +98,77 @@ describe('Transaction duration anomaly alert', () => { } as unknown) as MlPluginSetup; registerTransactionDurationAnomalyAlertType({ - alerting, + ...dependencies, ml, - config$: mockedConfig$, }); - expect(alertExecutor).toBeDefined(); - const scheduleActions = jest.fn(); - const services = { - callCluster: jest.fn(), - alertInstanceFactory: jest.fn(() => ({ scheduleActions })), - }; const params = { anomalySeverityType: ANOMALY_SEVERITY.MINOR }; - await alertExecutor!({ services, params }); - - await alertExecutor!({ services, params }); - [ - 'apm.transaction_duration_anomaly_foo_production_type-foo', - 'apm.transaction_duration_anomaly_bar_production_type-bar', - ].forEach((instanceName) => - expect(services.alertInstanceFactory).toHaveBeenCalledWith(instanceName) - ); + await executor({ params }); - expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { - serviceName: 'foo', - transactionType: 'type-foo', - environment: 'production', - threshold: 'minor', - thresholdValue: 'critical', - }); - expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { - serviceName: 'bar', - transactionType: 'type-bar', - environment: 'production', - threshold: 'minor', - thresholdValue: 'warning', - }); + expect( + services.scopedClusterClient.asCurrentUser.search + ).not.toHaveBeenCalled(); + expect(services.alertInstanceFactory).not.toHaveBeenCalled(); }); + }); - it('with service name', async () => { + describe('sends alert', () => { + it('for all services that exceeded the threshold', async () => { jest.spyOn(GetServiceAnomalies, 'getMLJobs').mockReturnValue( - Promise.resolve([ + Promise.resolve(([ { job_id: '1', - custom_settings: { - job_tags: { - environment: 'production', - }, - }, - } as unknown, + custom_settings: { job_tags: { environment: 'development' } }, + }, { job_id: '2', - custom_settings: { - job_tags: { - environment: 'testing', - }, - }, - } as unknown, - ] as Job[]) + custom_settings: { job_tags: { environment: 'production' } }, + }, + ] as unknown) as Job[]) ); - let alertExecutor: any; - - const alerting = { - registerType: ({ executor }) => { - alertExecutor = executor; - }, - } as AlertingPlugin['setup']; + const { + services, + dependencies, + executor, + scheduleActions, + } = createRuleTypeMocks(); const ml = ({ mlSystemProvider: () => ({ mlAnomalySearch: () => ({ - hits: { total: { value: 2 } }, aggregations: { - services: { + anomaly_groups: { buckets: [ - { key: 'foo', record_avg: { value: 80 } }, - { key: 'bar', record_avg: { value: 20 } }, + { + latest_score: { + top: [ + { + metrics: { + record_score: 80, + job_id: '1', + partition_field_value: 'foo', + by_field_value: 'type-foo', + }, + }, + ], + }, + }, + { + latest_score: { + top: [ + { + metrics: { + record_score: 20, + job_id: '2', + parttition_field_value: 'bar', + by_field_value: 'type-bar', + }, + }, + ], + }, + }, ], }, }, @@ -286,58 +178,26 @@ describe('Transaction duration anomaly alert', () => { } as unknown) as MlPluginSetup; registerTransactionDurationAnomalyAlertType({ - alerting, + ...dependencies, ml, - config$: mockedConfig$, }); - expect(alertExecutor).toBeDefined(); - const scheduleActions = jest.fn(); - const services = { - callCluster: jest.fn(), - alertInstanceFactory: jest.fn(() => ({ scheduleActions })), - }; const params = { anomalySeverityType: ANOMALY_SEVERITY.MINOR }; - await alertExecutor!({ services, params }); + await executor({ params }); + + expect(services.alertInstanceFactory).toHaveBeenCalledTimes(1); - await alertExecutor!({ services, params }); - [ - 'apm.transaction_duration_anomaly_foo_production', - 'apm.transaction_duration_anomaly_foo_testing', - 'apm.transaction_duration_anomaly_bar_production', - 'apm.transaction_duration_anomaly_bar_testing', - ].forEach((instanceName) => - expect(services.alertInstanceFactory).toHaveBeenCalledWith(instanceName) + expect(services.alertInstanceFactory).toHaveBeenCalledWith( + 'apm.transaction_duration_anomaly_foo_development_type-foo' ); expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { serviceName: 'foo', - transactionType: undefined, - environment: 'production', - threshold: 'minor', - thresholdValue: 'critical', - }); - expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { - serviceName: 'bar', - transactionType: undefined, - environment: 'production', - threshold: 'minor', - thresholdValue: 'warning', - }); - expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { - serviceName: 'foo', - transactionType: undefined, - environment: 'testing', - threshold: 'minor', - thresholdValue: 'critical', - }); - expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { - serviceName: 'bar', - transactionType: undefined, - environment: 'testing', + transactionType: 'type-foo', + environment: 'development', threshold: 'minor', - thresholdValue: 'warning', + triggerValue: 'critical', }); }); }); diff --git a/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.ts b/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.ts index 84c3ec7325fd2..15f4a8ea07801 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.ts @@ -6,9 +6,16 @@ */ import { schema } from '@kbn/config-schema'; -import { Observable } from 'rxjs'; -import { isEmpty } from 'lodash'; +import { compact } from 'lodash'; +import { ESSearchResponse } from 'typings/elasticsearch'; +import { QueryContainer } from '@elastic/elasticsearch/api/types'; import { getSeverity } from '../../../common/anomaly_detection'; +import { + SERVICE_ENVIRONMENT, + SERVICE_NAME, + TRANSACTION_TYPE, +} from '../../../common/elasticsearch_fieldnames'; +import { asMutableArray } from '../../../common/utils/as_mutable_array'; import { ANOMALY_SEVERITY } from '../../../../ml/common'; import { KibanaRequest } from '../../../../../../src/core/server'; import { @@ -16,17 +23,11 @@ import { ALERT_TYPES_CONFIG, ANOMALY_ALERT_SEVERITY_TYPES, } from '../../../common/alert_types'; -import { AlertingPlugin } from '../../../../alerting/server'; -import { APMConfig } from '../..'; -import { MlPluginSetup } from '../../../../ml/server'; import { getMLJobs } from '../service_map/get_service_anomalies'; import { apmActionVariables } from './action_variables'; - -interface RegisterAlertParams { - alerting: AlertingPlugin['setup']; - ml?: MlPluginSetup; - config$: Observable; -} +import { RegisterRuleDependencies } from './register_apm_alerts'; +import { parseEnvironmentUrlParam } from '../../../common/environment_filter_values'; +import { createAPMLifecycleRuleType } from './create_apm_lifecycle_rule_type'; const paramsSchema = schema.object({ serviceName: schema.maybe(schema.string()), @@ -46,203 +47,199 @@ const alertTypeConfig = ALERT_TYPES_CONFIG[AlertType.TransactionDurationAnomaly]; export function registerTransactionDurationAnomalyAlertType({ - alerting, + registry, ml, - config$, -}: RegisterAlertParams) { - alerting.registerType({ - id: AlertType.TransactionDurationAnomaly, - name: alertTypeConfig.name, - actionGroups: alertTypeConfig.actionGroups, - defaultActionGroupId: alertTypeConfig.defaultActionGroupId, - validate: { - params: paramsSchema, - }, - actionVariables: { - context: [ - apmActionVariables.serviceName, - apmActionVariables.transactionType, - apmActionVariables.environment, - apmActionVariables.threshold, - apmActionVariables.triggerValue, - ], - }, - producer: 'apm', - minimumLicenseRequired: 'basic', - executor: async ({ services, params, state }) => { - if (!ml) { - return; - } - const alertParams = params; - const request = {} as KibanaRequest; - const { mlAnomalySearch } = ml.mlSystemProvider( - request, - services.savedObjectsClient - ); - const anomalyDetectors = ml.anomalyDetectorsProvider( - request, - services.savedObjectsClient - ); - - const mlJobs = await getMLJobs(anomalyDetectors, alertParams.environment); - - const selectedOption = ANOMALY_ALERT_SEVERITY_TYPES.find( - (option) => option.type === alertParams.anomalySeverityType - ); - - if (!selectedOption) { - throw new Error( - `Anomaly alert severity type ${alertParams.anomalySeverityType} is not supported.` + logger, +}: RegisterRuleDependencies) { + registry.registerType( + createAPMLifecycleRuleType({ + id: AlertType.TransactionDurationAnomaly, + name: alertTypeConfig.name, + actionGroups: alertTypeConfig.actionGroups, + defaultActionGroupId: alertTypeConfig.defaultActionGroupId, + validate: { + params: paramsSchema, + }, + actionVariables: { + context: [ + apmActionVariables.serviceName, + apmActionVariables.transactionType, + apmActionVariables.environment, + apmActionVariables.threshold, + apmActionVariables.triggerValue, + ], + }, + producer: 'apm', + minimumLicenseRequired: 'basic', + executor: async ({ services, params }) => { + if (!ml) { + return {}; + } + const alertParams = params; + const request = {} as KibanaRequest; + const { mlAnomalySearch } = ml.mlSystemProvider( + request, + services.savedObjectsClient + ); + const anomalyDetectors = ml.anomalyDetectorsProvider( + request, + services.savedObjectsClient ); - } - const threshold = selectedOption.threshold; + const mlJobs = await getMLJobs( + anomalyDetectors, + alertParams.environment + ); - if (mlJobs.length === 0) { - return {}; - } - - const jobIds = mlJobs.map((job) => job.job_id); - const anomalySearchParams = { - terminateAfter: 1, - body: { - size: 0, - query: { - bool: { - filter: [ - { term: { result_type: 'record' } }, - { terms: { job_id: jobIds } }, - { - range: { - timestamp: { - gte: `now-${alertParams.windowSize}${alertParams.windowUnit}`, - format: 'epoch_millis', + const selectedOption = ANOMALY_ALERT_SEVERITY_TYPES.find( + (option) => option.type === alertParams.anomalySeverityType + ); + + if (!selectedOption) { + throw new Error( + `Anomaly alert severity type ${alertParams.anomalySeverityType} is not supported.` + ); + } + + const threshold = selectedOption.threshold; + + if (mlJobs.length === 0) { + return {}; + } + + const jobIds = mlJobs.map((job) => job.job_id); + const anomalySearchParams = { + body: { + size: 0, + query: { + bool: { + filter: [ + { term: { result_type: 'record' } }, + { terms: { job_id: jobIds } }, + { term: { is_interim: false } }, + { + range: { + timestamp: { + gte: `now-${alertParams.windowSize}${alertParams.windowUnit}`, + format: 'epoch_millis', + }, }, }, - }, - ...(alertParams.serviceName - ? [ - { - term: { - partition_field_value: alertParams.serviceName, + ...(alertParams.serviceName + ? [ + { + term: { + partition_field_value: alertParams.serviceName, + }, }, - }, - ] - : []), - ...(alertParams.transactionType - ? [ - { - term: { - by_field_value: alertParams.transactionType, + ] + : []), + ...(alertParams.transactionType + ? [ + { + term: { + by_field_value: alertParams.transactionType, + }, }, - }, - ] - : []), - { - range: { - record_score: { - gte: threshold, - }, - }, - }, - ], - }, - }, - aggs: { - services: { - terms: { - field: 'partition_field_value', - size: 50, + ] + : []), + ] as QueryContainer[], }, - aggs: { - transaction_types: { - terms: { - field: 'by_field_value', - }, + }, + aggs: { + anomaly_groups: { + multi_terms: { + terms: [ + { field: 'partition_field_value' }, + { field: 'by_field_value' }, + { field: 'job_id' }, + ], + size: 10000, }, - record_avg: { - avg: { - field: 'record_score', + aggs: { + latest_score: { + top_metrics: { + metrics: asMutableArray([ + { field: 'record_score' }, + { field: 'partition_field_value' }, + { field: 'by_field_value' }, + { field: 'job_id' }, + ] as const), + sort: { + '@timestamp': 'desc' as const, + }, + }, }, }, }, }, }, - }, - }; - - const response = ((await mlAnomalySearch( - anomalySearchParams, - jobIds - )) as unknown) as { - hits: { total: { value: number } }; - aggregations?: { - services: { - buckets: Array<{ - key: string; - record_avg: { value: number }; - transaction_types: { buckets: Array<{ key: string }> }; - }>; - }; }; - }; - - const hitCount = response.hits.total.value; - - if (hitCount > 0) { - function scheduleAction({ - serviceName, - severity, - environment, - transactionType, - }: { - serviceName: string; - severity: string; - environment?: string; - transactionType?: string; - }) { - const alertInstanceName = [ - AlertType.TransactionDurationAnomaly, - serviceName, - environment, - transactionType, - ] - .filter((name) => name) - .join('_'); - - const alertInstance = services.alertInstanceFactory( - alertInstanceName - ); - alertInstance.scheduleActions(alertTypeConfig.defaultActionGroupId, { - serviceName, - environment, - transactionType, - threshold: selectedOption?.label, - thresholdValue: severity, - }); - } - mlJobs.map((job) => { - const environment = job.custom_settings?.job_tags?.environment; - response.aggregations?.services.buckets.forEach((serviceBucket) => { - const serviceName = serviceBucket.key as string; - const severity = getSeverity(serviceBucket.record_avg.value); - if (isEmpty(serviceBucket.transaction_types?.buckets)) { - scheduleAction({ serviceName, severity, environment }); - } else { - serviceBucket.transaction_types?.buckets.forEach((typeBucket) => { - const transactionType = typeBucket.key as string; - scheduleAction({ - serviceName, - severity, - environment, - transactionType, - }); - }); - } - }); + const response: ESSearchResponse< + unknown, + typeof anomalySearchParams + > = (await mlAnomalySearch(anomalySearchParams, [])) as any; + + const anomalies = + response.aggregations?.anomaly_groups.buckets + .map((bucket) => { + const latest = bucket.latest_score.top[0].metrics; + + const job = mlJobs.find((j) => j.job_id === latest.job_id); + + if (!job) { + logger.warn( + `Could not find matching job for job id ${latest.job_id}` + ); + return undefined; + } + + return { + serviceName: latest.partition_field_value as string, + transactionType: latest.by_field_value as string, + environment: job.custom_settings!.job_tags!.environment, + score: latest.record_score as number, + }; + }) + .filter((anomaly) => + anomaly ? anomaly.score >= threshold : false + ) ?? []; + + compact(anomalies).forEach((anomaly) => { + const { serviceName, environment, transactionType, score } = anomaly; + + const parsedEnvironment = parseEnvironmentUrlParam(environment); + + services + .alertWithLifecycle({ + id: [ + AlertType.TransactionDurationAnomaly, + serviceName, + environment, + transactionType, + ] + .filter((name) => name) + .join('_'), + fields: { + [SERVICE_NAME]: serviceName, + ...(parsedEnvironment.esFieldValue + ? { [SERVICE_ENVIRONMENT]: environment } + : {}), + [TRANSACTION_TYPE]: transactionType, + }, + }) + .scheduleActions(alertTypeConfig.defaultActionGroupId, { + serviceName, + transactionType, + environment, + threshold: selectedOption?.label, + triggerValue: getSeverity(score), + }); }); - } - }, - }); + + return {}; + }, + }) + ); } diff --git a/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.test.ts b/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.test.ts index 148cd813a8a22..be5f4705482d0 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.test.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.test.ts @@ -5,48 +5,19 @@ * 2.0. */ -import { Observable } from 'rxjs'; -import * as Rx from 'rxjs'; -import { toArray, map } from 'rxjs/operators'; -import { AlertingPlugin } from '../../../../alerting/server'; -import { APMConfig } from '../..'; import { registerTransactionErrorRateAlertType } from './register_transaction_error_rate_alert_type'; -import { elasticsearchServiceMock } from 'src/core/server/mocks'; // eslint-disable-next-line @kbn/eslint/no-restricted-paths import { elasticsearchClientMock } from 'src/core/server/elasticsearch/client/mocks'; - -type Operator = (source: Rx.Observable) => Rx.Observable; -const pipeClosure = (fn: Operator): Operator => { - return (source: Rx.Observable) => { - return Rx.defer(() => fn(source)); - }; -}; -const mockedConfig$ = (Rx.of('apm_oss.errorIndices').pipe( - pipeClosure((source$) => { - return source$.pipe(map((i) => i)); - }), - toArray() -) as unknown) as Observable; +import { createRuleTypeMocks } from './test_utils'; describe('Transaction error rate alert', () => { it("doesn't send an alert when rate is less than threshold", async () => { - let alertExecutor: any; - const alerting = { - registerType: ({ executor }) => { - alertExecutor = executor; - }, - } as AlertingPlugin['setup']; + const { services, dependencies, executor } = createRuleTypeMocks(); registerTransactionErrorRateAlertType({ - alerting, - config$: mockedConfig$, + ...dependencies, }); - expect(alertExecutor).toBeDefined(); - const services = { - scopedClusterClient: elasticsearchServiceMock.createScopedClusterClient(), - alertInstanceFactory: jest.fn(), - }; const params = { threshold: 1 }; services.scopedClusterClient.asCurrentUser.search.mockReturnValue( @@ -60,6 +31,11 @@ describe('Transaction error rate alert', () => { }, took: 0, timed_out: false, + aggregations: { + series: { + buckets: [], + }, + }, _shards: { failed: 0, skipped: 0, @@ -69,30 +45,21 @@ describe('Transaction error rate alert', () => { }) ); - await alertExecutor!({ services, params }); + await executor({ params }); expect(services.alertInstanceFactory).not.toBeCalled(); }); - it('sends alerts with service name, transaction type and environment', async () => { - let alertExecutor: any; - const alerting = { - registerType: ({ executor }) => { - alertExecutor = executor; - }, - } as AlertingPlugin['setup']; + it('sends alerts for services that exceeded the threshold', async () => { + const { + services, + dependencies, + executor, + scheduleActions, + } = createRuleTypeMocks(); registerTransactionErrorRateAlertType({ - alerting, - config$: mockedConfig$, + ...dependencies, }); - expect(alertExecutor).toBeDefined(); - - const scheduleActions = jest.fn(); - const services = { - scopedClusterClient: elasticsearchServiceMock.createScopedClusterClient(), - alertInstanceFactory: jest.fn(() => ({ scheduleActions })), - }; - const params = { threshold: 10, windowSize: 5, windowUnit: 'm' }; services.scopedClusterClient.asCurrentUser.search.mockReturnValue( elasticsearchClientMock.createSuccessTransportRequestPromise({ @@ -100,37 +67,38 @@ describe('Transaction error rate alert', () => { hits: [], total: { relation: 'eq', - value: 4, + value: 0, }, }, aggregations: { - failed_transactions: { - doc_count: 2, - }, - services: { + series: { buckets: [ { - key: 'foo', - transaction_types: { + key: ['foo', 'env-foo', 'type-foo'], + outcomes: { buckets: [ { - key: 'type-foo', - environments: { - buckets: [{ key: 'env-foo' }, { key: 'env-foo-2' }], - }, + key: 'success', + doc_count: 90, + }, + { + key: 'failure', + doc_count: 10, }, ], }, }, { - key: 'bar', - transaction_types: { + key: ['bar', 'env-bar', 'type-bar'], + outcomes: { buckets: [ { - key: 'type-bar', - environments: { - buckets: [{ key: 'env-bar' }, { key: 'env-bar-2' }], - }, + key: 'success', + doc_count: 90, + }, + { + key: 'failure', + doc_count: 1, }, ], }, @@ -149,208 +117,25 @@ describe('Transaction error rate alert', () => { }) ); - await alertExecutor!({ services, params }); - [ - 'apm.transaction_error_rate_foo_type-foo_env-foo', - 'apm.transaction_error_rate_foo_type-foo_env-foo-2', - 'apm.transaction_error_rate_bar_type-bar_env-bar', - 'apm.transaction_error_rate_bar_type-bar_env-bar-2', - ].forEach((instanceName) => - expect(services.alertInstanceFactory).toHaveBeenCalledWith(instanceName) - ); - - expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { - serviceName: 'foo', - transactionType: 'type-foo', - environment: 'env-foo', - threshold: 10, - triggerValue: '50', - interval: '5m', - }); - expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { - serviceName: 'foo', - transactionType: 'type-foo', - environment: 'env-foo-2', - threshold: 10, - triggerValue: '50', - interval: '5m', - }); - expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { - serviceName: 'bar', - transactionType: 'type-bar', - environment: 'env-bar', - threshold: 10, - triggerValue: '50', - interval: '5m', - }); - expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { - serviceName: 'bar', - transactionType: 'type-bar', - environment: 'env-bar-2', - threshold: 10, - triggerValue: '50', - interval: '5m', - }); - }); - it('sends alerts with service name and transaction type', async () => { - let alertExecutor: any; - const alerting = { - registerType: ({ executor }) => { - alertExecutor = executor; - }, - } as AlertingPlugin['setup']; - - registerTransactionErrorRateAlertType({ - alerting, - config$: mockedConfig$, - }); - expect(alertExecutor).toBeDefined(); - - const scheduleActions = jest.fn(); - const services = { - scopedClusterClient: elasticsearchServiceMock.createScopedClusterClient(), - alertInstanceFactory: jest.fn(() => ({ scheduleActions })), - }; const params = { threshold: 10, windowSize: 5, windowUnit: 'm' }; - services.scopedClusterClient.asCurrentUser.search.mockReturnValue( - elasticsearchClientMock.createSuccessTransportRequestPromise({ - hits: { - hits: [], - total: { - relation: 'eq', - value: 4, - }, - }, - aggregations: { - failed_transactions: { - doc_count: 2, - }, - services: { - buckets: [ - { - key: 'foo', - transaction_types: { - buckets: [{ key: 'type-foo' }], - }, - }, - { - key: 'bar', - transaction_types: { - buckets: [{ key: 'type-bar' }], - }, - }, - ], - }, - }, - took: 0, - timed_out: false, - _shards: { - failed: 0, - skipped: 0, - successful: 1, - total: 1, - }, - }) - ); - - await alertExecutor!({ services, params }); - [ - 'apm.transaction_error_rate_foo_type-foo', - 'apm.transaction_error_rate_bar_type-bar', - ].forEach((instanceName) => - expect(services.alertInstanceFactory).toHaveBeenCalledWith(instanceName) - ); - - expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { - serviceName: 'foo', - transactionType: 'type-foo', - environment: undefined, - threshold: 10, - triggerValue: '50', - interval: '5m', - }); - expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { - serviceName: 'bar', - transactionType: 'type-bar', - environment: undefined, - threshold: 10, - triggerValue: '50', - interval: '5m', - }); - }); - - it('sends alerts with service name', async () => { - let alertExecutor: any; - const alerting = { - registerType: ({ executor }) => { - alertExecutor = executor; - }, - } as AlertingPlugin['setup']; - - registerTransactionErrorRateAlertType({ - alerting, - config$: mockedConfig$, - }); - expect(alertExecutor).toBeDefined(); + await executor({ params }); - const scheduleActions = jest.fn(); - const services = { - scopedClusterClient: elasticsearchServiceMock.createScopedClusterClient(), - alertInstanceFactory: jest.fn(() => ({ scheduleActions })), - }; - const params = { threshold: 10, windowSize: 5, windowUnit: 'm' }; + expect(services.alertInstanceFactory).toHaveBeenCalledTimes(1); - services.scopedClusterClient.asCurrentUser.search.mockReturnValue( - elasticsearchClientMock.createSuccessTransportRequestPromise({ - hits: { - hits: [], - total: { - value: 4, - relation: 'eq', - }, - }, - aggregations: { - failed_transactions: { - doc_count: 2, - }, - services: { - buckets: [{ key: 'foo' }, { key: 'bar' }], - }, - }, - took: 0, - timed_out: false, - _shards: { - failed: 0, - skipped: 0, - successful: 1, - total: 1, - }, - }) + expect(services.alertInstanceFactory).toHaveBeenCalledWith( + 'apm.transaction_error_rate_foo_type-foo_env-foo' ); - - await alertExecutor!({ services, params }); - [ - 'apm.transaction_error_rate_foo', - 'apm.transaction_error_rate_bar', - ].forEach((instanceName) => - expect(services.alertInstanceFactory).toHaveBeenCalledWith(instanceName) + expect(services.alertInstanceFactory).not.toHaveBeenCalledWith( + 'apm.transaction_error_rate_bar_type-bar_env-bar' ); expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { serviceName: 'foo', - transactionType: undefined, - environment: undefined, - threshold: 10, - triggerValue: '50', - interval: '5m', - }); - expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { - serviceName: 'bar', - transactionType: undefined, - environment: undefined, + transactionType: 'type-foo', + environment: 'env-foo', threshold: 10, - triggerValue: '50', + triggerValue: '10', interval: '5m', }); }); diff --git a/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.ts b/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.ts index 0b2684cdaf083..0865bed41142e 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.ts @@ -6,11 +6,7 @@ */ import { schema } from '@kbn/config-schema'; -import { isEmpty } from 'lodash'; -import { Observable } from 'rxjs'; import { take } from 'rxjs/operators'; -import { APMConfig } from '../..'; -import { AlertingPlugin } from '../../../../alerting/server'; import { AlertType, ALERT_TYPES_CONFIG } from '../../../common/alert_types'; import { EVENT_OUTCOME, @@ -26,11 +22,8 @@ import { environmentQuery } from '../../../server/utils/queries'; import { getApmIndices } from '../settings/apm_indices/get_apm_indices'; import { apmActionVariables } from './action_variables'; import { alertingEsClient } from './alerting_es_client'; - -interface RegisterAlertParams { - alerting: AlertingPlugin['setup']; - config$: Observable; -} +import { createAPMLifecycleRuleType } from './create_apm_lifecycle_rule_type'; +import { RegisterRuleDependencies } from './register_apm_alerts'; const paramsSchema = schema.object({ windowSize: schema.number(), @@ -44,158 +37,165 @@ const paramsSchema = schema.object({ const alertTypeConfig = ALERT_TYPES_CONFIG[AlertType.TransactionErrorRate]; export function registerTransactionErrorRateAlertType({ - alerting, + registry, config$, -}: RegisterAlertParams) { - alerting.registerType({ - id: AlertType.TransactionErrorRate, - name: alertTypeConfig.name, - actionGroups: alertTypeConfig.actionGroups, - defaultActionGroupId: alertTypeConfig.defaultActionGroupId, - validate: { - params: paramsSchema, - }, - actionVariables: { - context: [ - apmActionVariables.transactionType, - apmActionVariables.serviceName, - apmActionVariables.environment, - apmActionVariables.threshold, - apmActionVariables.triggerValue, - apmActionVariables.interval, - ], - }, - producer: 'apm', - minimumLicenseRequired: 'basic', - executor: async ({ services, params: alertParams }) => { - const config = await config$.pipe(take(1)).toPromise(); - const indices = await getApmIndices({ - config, - savedObjectsClient: services.savedObjectsClient, - }); - const maxServiceEnvironments = config['xpack.apm.maxServiceEnvironments']; +}: RegisterRuleDependencies) { + registry.registerType( + createAPMLifecycleRuleType({ + id: AlertType.TransactionErrorRate, + name: alertTypeConfig.name, + actionGroups: alertTypeConfig.actionGroups, + defaultActionGroupId: alertTypeConfig.defaultActionGroupId, + validate: { + params: paramsSchema, + }, + actionVariables: { + context: [ + apmActionVariables.transactionType, + apmActionVariables.serviceName, + apmActionVariables.environment, + apmActionVariables.threshold, + apmActionVariables.triggerValue, + apmActionVariables.interval, + ], + }, + producer: 'apm', + minimumLicenseRequired: 'basic', + executor: async ({ services, params: alertParams }) => { + const config = await config$.pipe(take(1)).toPromise(); + const indices = await getApmIndices({ + config, + savedObjectsClient: services.savedObjectsClient, + }); - const searchParams = { - index: indices['apm_oss.transactionIndices'], - size: 0, - body: { - track_total_hits: true, - query: { - bool: { - filter: [ - { - range: { - '@timestamp': { - gte: `now-${alertParams.windowSize}${alertParams.windowUnit}`, + const searchParams = { + index: indices['apm_oss.transactionIndices'], + size: 1, + body: { + query: { + bool: { + filter: [ + { + range: { + '@timestamp': { + gte: `now-${alertParams.windowSize}${alertParams.windowUnit}`, + }, }, }, - }, - { term: { [PROCESSOR_EVENT]: ProcessorEvent.transaction } }, - ...(alertParams.serviceName - ? [{ term: { [SERVICE_NAME]: alertParams.serviceName } }] - : []), - ...(alertParams.transactionType - ? [ - { - term: { - [TRANSACTION_TYPE]: alertParams.transactionType, + { term: { [PROCESSOR_EVENT]: ProcessorEvent.transaction } }, + { + terms: { + [EVENT_OUTCOME]: [ + EventOutcome.failure, + EventOutcome.success, + ], + }, + }, + ...(alertParams.serviceName + ? [{ term: { [SERVICE_NAME]: alertParams.serviceName } }] + : []), + ...(alertParams.transactionType + ? [ + { + term: { + [TRANSACTION_TYPE]: alertParams.transactionType, + }, }, - }, - ] - : []), - ...environmentQuery(alertParams.environment), - ], - }, - }, - aggs: { - failed_transactions: { - filter: { term: { [EVENT_OUTCOME]: EventOutcome.failure } }, - }, - services: { - terms: { - field: SERVICE_NAME, - size: 50, + ] + : []), + ...environmentQuery(alertParams.environment), + ], }, - aggs: { - transaction_types: { - terms: { field: TRANSACTION_TYPE }, - aggs: { - environments: { - terms: { - field: SERVICE_ENVIRONMENT, - size: maxServiceEnvironments, - }, + }, + aggs: { + series: { + multi_terms: { + terms: [ + { field: SERVICE_NAME }, + { field: SERVICE_ENVIRONMENT, missing: '' }, + { field: TRANSACTION_TYPE }, + ], + size: 10000, + }, + aggs: { + outcomes: { + terms: { + field: EVENT_OUTCOME, }, }, }, }, }, }, - }, - }; + }; - const { body: response } = await alertingEsClient(services, searchParams); - if (!response.aggregations) { - return; - } + const response = await alertingEsClient( + services.scopedClusterClient, + searchParams + ); - const failedTransactionCount = - response.aggregations.failed_transactions.doc_count; - const totalTransactionCount = response.hits.total.value; - const transactionErrorRate = - (failedTransactionCount / totalTransactionCount) * 100; + if (!response.aggregations) { + return {}; + } - if (transactionErrorRate > alertParams.threshold) { - function scheduleAction({ - serviceName, - environment, - transactionType, - }: { - serviceName: string; - environment?: string; - transactionType?: string; - }) { - const alertInstanceName = [ - AlertType.TransactionErrorRate, - serviceName, - transactionType, - environment, - ] - .filter((name) => name) - .join('_'); + const results = response.aggregations.series.buckets + .map((bucket) => { + const [serviceName, environment, transactionType] = bucket.key; + + const failed = + bucket.outcomes.buckets.find( + (outcomeBucket) => outcomeBucket.key === EventOutcome.failure + )?.doc_count ?? 0; + const succesful = + bucket.outcomes.buckets.find( + (outcomeBucket) => outcomeBucket.key === EventOutcome.success + )?.doc_count ?? 0; - const alertInstance = services.alertInstanceFactory( - alertInstanceName - ); - alertInstance.scheduleActions(alertTypeConfig.defaultActionGroupId, { + return { + serviceName, + environment, + transactionType, + errorRate: (failed / (failed + succesful)) * 100, + }; + }) + .filter((result) => result.errorRate >= alertParams.threshold); + + results.forEach((result) => { + const { serviceName, - transactionType, environment, - threshold: alertParams.threshold, - triggerValue: asDecimalOrInteger(transactionErrorRate), - interval: `${alertParams.windowSize}${alertParams.windowUnit}`, - }); - } + transactionType, + errorRate, + } = result; - response.aggregations?.services.buckets.forEach((serviceBucket) => { - const serviceName = serviceBucket.key as string; - if (isEmpty(serviceBucket.transaction_types?.buckets)) { - scheduleAction({ serviceName }); - } else { - serviceBucket.transaction_types.buckets.forEach((typeBucket) => { - const transactionType = typeBucket.key as string; - if (isEmpty(typeBucket.environments?.buckets)) { - scheduleAction({ serviceName, transactionType }); - } else { - typeBucket.environments.buckets.forEach((envBucket) => { - const environment = envBucket.key as string; - scheduleAction({ serviceName, transactionType, environment }); - }); - } + services + .alertWithLifecycle({ + id: [ + AlertType.TransactionErrorRate, + serviceName, + transactionType, + environment, + ] + .filter((name) => name) + .join('_'), + fields: { + [SERVICE_NAME]: serviceName, + ...(environment ? { [SERVICE_ENVIRONMENT]: environment } : {}), + [TRANSACTION_TYPE]: transactionType, + }, + }) + .scheduleActions(alertTypeConfig.defaultActionGroupId, { + serviceName, + transactionType, + environment, + threshold: alertParams.threshold, + triggerValue: asDecimalOrInteger(errorRate), + interval: `${alertParams.windowSize}${alertParams.windowUnit}`, }); - } }); - } - }, - }); + + return {}; + }, + }) + ); } diff --git a/x-pack/plugins/apm/server/lib/alerts/test_utils/index.ts b/x-pack/plugins/apm/server/lib/alerts/test_utils/index.ts new file mode 100644 index 0000000000000..37b3e282d0a59 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/alerts/test_utils/index.ts @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Logger } from 'kibana/server'; +import { of } from 'rxjs'; +import { elasticsearchServiceMock } from 'src/core/server/mocks'; +import { APMConfig } from '../../..'; +import { APMRuleRegistry } from '../../../plugin'; + +export const createRuleTypeMocks = () => { + let alertExecutor: (...args: any[]) => Promise; + + const mockedConfig$ = of({ + /* eslint-disable @typescript-eslint/naming-convention */ + 'apm_oss.errorIndices': 'apm-*', + 'apm_oss.transactionIndices': 'apm-*', + /* eslint-enable @typescript-eslint/naming-convention */ + } as APMConfig); + + const loggerMock = ({ + debug: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + } as unknown) as Logger; + + const registry = { + registerType: ({ executor }) => { + alertExecutor = executor; + }, + } as APMRuleRegistry; + + const scheduleActions = jest.fn(); + + const services = { + scopedClusterClient: elasticsearchServiceMock.createScopedClusterClient(), + scopedRuleRegistryClient: { + bulkIndex: jest.fn(), + }, + alertInstanceFactory: jest.fn(() => ({ scheduleActions })), + alertWithLifecycle: jest.fn(), + logger: loggerMock, + }; + + return { + dependencies: { + registry, + config$: mockedConfig$, + logger: loggerMock, + }, + services, + scheduleActions, + executor: async ({ params }: { params: Record }) => { + return alertExecutor({ + services, + params, + startedAt: new Date(), + }); + }, + }; +}; diff --git a/x-pack/plugins/apm/server/plugin.ts b/x-pack/plugins/apm/server/plugin.ts index 074df7eaafd3c..cb94b18a1ecf9 100644 --- a/x-pack/plugins/apm/server/plugin.ts +++ b/x-pack/plugins/apm/server/plugin.ts @@ -43,6 +43,8 @@ import { import { registerRoutes } from './routes/register_routes'; import { getGlobalApmServerRouteRepository } from './routes/get_global_apm_server_route_repository'; +export type APMRuleRegistry = ReturnType['ruleRegistry']; + export class APMPlugin implements Plugin< @@ -72,15 +74,6 @@ export class APMPlugin core.uiSettings.register(uiSettings); - if (plugins.actions && plugins.alerting) { - registerApmAlerts({ - alerting: plugins.alerting, - actions: plugins.actions, - ml: plugins.ml, - config$: mergedConfig$, - }); - } - const currentConfig = mergeConfigs( plugins.apmOss.config, this.initContext.config.get() @@ -157,6 +150,28 @@ export class APMPlugin config: await mergedConfig$.pipe(take(1)).toPromise(), }); + const apmRuleRegistry = plugins.observability.ruleRegistry.create({ + name: 'apm', + fieldMap: { + 'service.environment': { + type: 'keyword', + }, + 'transaction.type': { + type: 'keyword', + }, + 'processor.event': { + type: 'keyword', + }, + }, + }); + + registerApmAlerts({ + registry: apmRuleRegistry, + ml: plugins.ml, + config$: mergedConfig$, + logger: this.logger!.get('rule'), + }); + return { config$: mergedConfig$, getApmIndices: boundGetApmIndices, @@ -186,6 +201,7 @@ export class APMPlugin }, }); }, + ruleRegistry: apmRuleRegistry, }; } diff --git a/x-pack/plugins/apm/server/routes/typings.ts b/x-pack/plugins/apm/server/routes/typings.ts index 0fec88a4326c3..517387c5f74ef 100644 --- a/x-pack/plugins/apm/server/routes/typings.ts +++ b/x-pack/plugins/apm/server/routes/typings.ts @@ -60,21 +60,3 @@ export interface APMRouteHandlerResources { }; }; } - -// export type Client< -// TRouteState, -// TOptions extends { abortable: boolean } = { abortable: true } -// > = ( -// options: Omit< -// FetchOptions, -// 'query' | 'body' | 'pathname' | 'method' | 'signal' -// > & { -// forceCache?: boolean; -// endpoint: TEndpoint; -// } & MaybeParams & -// (TOptions extends { abortable: true } ? { signal: AbortSignal | null } : {}) -// ) => Promise< -// TRouteState[TEndpoint] extends { ret: any } -// ? TRouteState[TEndpoint]['ret'] -// : unknown -// >; diff --git a/x-pack/plugins/apm/server/types.ts b/x-pack/plugins/apm/server/types.ts index cef9eaf2f4fc0..dbc220f9f6b15 100644 --- a/x-pack/plugins/apm/server/types.ts +++ b/x-pack/plugins/apm/server/types.ts @@ -125,6 +125,7 @@ const requiredDependencies = [ 'triggersActionsUi', 'embeddable', 'infra', + 'observability', ] as const; const optionalDependencies = [ @@ -134,7 +135,6 @@ const optionalDependencies = [ 'taskManager', 'actions', 'alerting', - 'observability', 'security', 'ml', 'home', diff --git a/x-pack/plugins/apm/tsconfig.json b/x-pack/plugins/apm/tsconfig.json index ffbf11c23f63a..bb341059e2d43 100644 --- a/x-pack/plugins/apm/tsconfig.json +++ b/x-pack/plugins/apm/tsconfig.json @@ -38,6 +38,7 @@ { "path": "../ml/tsconfig.json" }, { "path": "../observability/tsconfig.json" }, { "path": "../reporting/tsconfig.json" }, + { "path": "../rule_registry/tsconfig.json" }, { "path": "../security/tsconfig.json" }, { "path": "../task_manager/tsconfig.json" }, { "path": "../triggers_actions_ui/tsconfig.json" } diff --git a/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts b/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts index efe3186f97805..cd87bc8e6e18b 100644 --- a/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts +++ b/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts @@ -12,7 +12,6 @@ import { IClusterClientAdapter, EVENT_BUFFER_LENGTH, } from './cluster_client_adapter'; -import { contextMock } from './context.mock'; import { findOptionsSchema } from '../event_log_client'; import { delay } from '../lib/delay'; import { times } from 'lodash'; @@ -31,7 +30,7 @@ beforeEach(() => { clusterClientAdapter = new ClusterClientAdapter({ logger, elasticsearchClientPromise: Promise.resolve(clusterClient), - context: contextMock.create(), + wait: () => Promise.resolve(true), }); }); diff --git a/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts b/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts index 5d7be2278d55d..dd6ac6350d6e3 100644 --- a/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts +++ b/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts @@ -10,8 +10,8 @@ import { bufferTime, filter as rxFilter, switchMap } from 'rxjs/operators'; import { reject, isUndefined, isNumber } from 'lodash'; import type { PublicMethodsOf } from '@kbn/utility-types'; import { Logger, ElasticsearchClient } from 'src/core/server'; +import util from 'util'; import { estypes } from '@elastic/elasticsearch'; -import { EsContext } from '.'; import { IEvent, IValidatedEvent, SAVED_OBJECT_REL_PRIMARY } from '../types'; import { FindOptionsType } from '../event_log_client'; import { esKuery } from '../../../../../src/plugins/data/server'; @@ -26,10 +26,12 @@ export interface Doc { body: IEvent; } +type Wait = () => Promise; + export interface ConstructorOpts { logger: Logger; elasticsearchClientPromise: Promise; - context: EsContext; + wait: Wait; } export interface QueryEventsBySavedObjectResult { @@ -39,18 +41,21 @@ export interface QueryEventsBySavedObjectResult { data: IValidatedEvent[]; } -export class ClusterClientAdapter { +// eslint-disable-next-line @typescript-eslint/no-explicit-any +type AliasAny = any; + +export class ClusterClientAdapter { private readonly logger: Logger; private readonly elasticsearchClientPromise: Promise; - private readonly docBuffer$: Subject; - private readonly context: EsContext; + private readonly docBuffer$: Subject; + private readonly wait: Wait; private readonly docsBufferedFlushed: Promise; constructor(opts: ConstructorOpts) { this.logger = opts.logger; this.elasticsearchClientPromise = opts.elasticsearchClientPromise; - this.context = opts.context; - this.docBuffer$ = new Subject(); + this.wait = opts.wait; + this.docBuffer$ = new Subject(); // buffer event log docs for time / buffer length, ignore empty // buffers, then index the buffered docs; kick things off with a @@ -75,18 +80,21 @@ export class ClusterClientAdapter { await this.docsBufferedFlushed; } - public indexDocument(doc: Doc): void { + public indexDocument(doc: TDoc): void { this.docBuffer$.next(doc); } - async indexDocuments(docs: Doc[]): Promise { + async indexDocuments(docs: TDoc[]): Promise { // If es initialization failed, don't try to index. // Also, don't log here, we log the failure case in plugin startup // instead, otherwise we'd be spamming the log (if done here) - if (!(await this.context.waitTillReady())) { + if (!(await this.wait())) { + this.logger.debug(`Initialization failed, not indexing ${docs.length} documents`); return; } + this.logger.debug(`Indexing ${docs.length} documents`); + const bulkBody: Array> = []; for (const doc of docs) { @@ -98,7 +106,13 @@ export class ClusterClientAdapter { try { const esClient = await this.elasticsearchClientPromise; - await esClient.bulk({ body: bulkBody }); + const response = await esClient.bulk({ body: bulkBody }); + + if (response.body.errors) { + const error = new Error('Error writing some bulk events'); + error.stack += '\n' + util.inspect(response.body.items, { depth: null }); + this.logger.error(error); + } } catch (err) { this.logger.error( `error writing bulk events: "${err.message}"; docs: ${JSON.stringify(bulkBody)}` @@ -156,7 +170,9 @@ export class ClusterClientAdapter { // instances at the same time. const existsNow = await this.doesIndexTemplateExist(name); if (!existsNow) { - throw new Error(`error creating index template: ${err.message}`); + const error = new Error(`error creating index template: ${err.message}`); + Object.assign(error, { wrapped: err }); + throw error; } } } diff --git a/x-pack/plugins/event_log/server/es/context.ts b/x-pack/plugins/event_log/server/es/context.ts index 6d3b2208b3408..f6ae0a2002dd4 100644 --- a/x-pack/plugins/event_log/server/es/context.ts +++ b/x-pack/plugins/event_log/server/es/context.ts @@ -53,7 +53,7 @@ class EsContextImpl implements EsContext { this.esAdapter = new ClusterClientAdapter({ logger: params.logger, elasticsearchClientPromise: params.elasticsearchClientPromise, - context: this, + wait: () => this.readySignal.wait(), }); } diff --git a/x-pack/plugins/event_log/server/index.ts b/x-pack/plugins/event_log/server/index.ts index ab2b9709703c5..4c5513a7fc59c 100644 --- a/x-pack/plugins/event_log/server/index.ts +++ b/x-pack/plugins/event_log/server/index.ts @@ -20,5 +20,9 @@ export { SAVED_OBJECT_REL_PRIMARY, } from './types'; +export { ClusterClientAdapter } from './es/cluster_client_adapter'; + +export { createReadySignal } from './lib/ready_signal'; + export const config = { schema: ConfigSchema }; export const plugin = (context: PluginInitializerContext) => new Plugin(context); diff --git a/x-pack/plugins/observability/kibana.json b/x-pack/plugins/observability/kibana.json index 5c47d0376581a..74efc1f4985a3 100644 --- a/x-pack/plugins/observability/kibana.json +++ b/x-pack/plugins/observability/kibana.json @@ -3,7 +3,7 @@ "version": "8.0.0", "kibanaVersion": "kibana", "configPath": ["xpack", "observability"], - "optionalPlugins": ["licensing", "home", "usageCollection","lens"], + "optionalPlugins": ["licensing", "home", "usageCollection","lens", "ruleRegistry"], "requiredPlugins": ["data"], "ui": true, "server": true, diff --git a/x-pack/plugins/observability/server/plugin.ts b/x-pack/plugins/observability/server/plugin.ts index cb9878f578885..c59b4dbe373dd 100644 --- a/x-pack/plugins/observability/server/plugin.ts +++ b/x-pack/plugins/observability/server/plugin.ts @@ -6,29 +6,30 @@ */ import { PluginInitializerContext, Plugin, CoreSetup } from 'src/core/server'; +import { pickWithPatterns } from '../../rule_registry/server'; import { ObservabilityConfig } from '.'; import { bootstrapAnnotations, - ScopedAnnotationsClient, ScopedAnnotationsClientFactory, AnnotationsAPI, } from './lib/annotations/bootstrap_annotations'; +import type { RuleRegistryPluginSetupContract } from '../../rule_registry/server'; import { uiSettings } from './ui_settings'; +import { ecsFieldMap } from '../../rule_registry/server'; -type LazyScopedAnnotationsClientFactory = ( - ...args: Parameters -) => Promise; - -export interface ObservabilityPluginSetup { - getScopedAnnotationsClient: LazyScopedAnnotationsClientFactory; -} +export type ObservabilityPluginSetup = ReturnType; export class ObservabilityPlugin implements Plugin { constructor(private readonly initContext: PluginInitializerContext) { this.initContext = initContext; } - public setup(core: CoreSetup, plugins: {}): ObservabilityPluginSetup { + public setup( + core: CoreSetup, + plugins: { + ruleRegistry: RuleRegistryPluginSetupContract; + } + ) { const config = this.initContext.config.get(); let annotationsApiPromise: Promise | undefined; @@ -48,10 +49,16 @@ export class ObservabilityPlugin implements Plugin { } return { - getScopedAnnotationsClient: async (...args) => { + getScopedAnnotationsClient: async (...args: Parameters) => { const api = await annotationsApiPromise; return api?.getScopedAnnotationsClient(...args); }, + ruleRegistry: plugins.ruleRegistry.create({ + name: 'observability', + fieldMap: { + ...pickWithPatterns(ecsFieldMap, 'host.name', 'service.name'), + }, + }), }; } diff --git a/x-pack/plugins/observability/tsconfig.json b/x-pack/plugins/observability/tsconfig.json index f55ae640a8026..bd37bc09bc130 100644 --- a/x-pack/plugins/observability/tsconfig.json +++ b/x-pack/plugins/observability/tsconfig.json @@ -23,6 +23,7 @@ { "path": "../../../src/plugins/kibana_utils/tsconfig.json" }, { "path": "../../../src/plugins/usage_collection/tsconfig.json" }, { "path": "../alerting/tsconfig.json" }, + { "path": "../rule_registry/tsconfig.json" }, { "path": "../licensing/tsconfig.json" }, { "path": "../lens/tsconfig.json" }, { "path": "../translations/tsconfig.json" } diff --git a/x-pack/plugins/rule_registry/README.md b/x-pack/plugins/rule_registry/README.md new file mode 100644 index 0000000000000..17fe2b20f74fa --- /dev/null +++ b/x-pack/plugins/rule_registry/README.md @@ -0,0 +1,68 @@ +The rule registry plugin aims to make it easy for rule type producers to have their rules produce the data that they need to build rich experiences on top of a unified experience, without the risk of mapping conflicts. + +A rule registry creates a template, an ILM policy, and an alias. The template mappings can be configured. It also injects a client scoped to these indices. + +It also supports inheritance, which means that producers can create a registry specific to their solution or rule type, and specify additional mappings to be used. + +The rule registry plugin creates a root rule registry, with the mappings defined needed to create a unified experience. Rule type producers can use the plugin to access the root rule registry, and create their own registry that branches off of the root rule registry. The rule registry client sees data from its own registry, and all registries that branches off of it. It does not see data from its parents. + +Creating a rule registry + +To create a rule registry, producers should add the `ruleRegistry` plugin to their dependencies. They can then use the `ruleRegistry.create` method to create a child registry, with the additional mappings that should be used by specifying `fieldMap`: + +```ts +const observabilityRegistry = plugins.ruleRegistry.create({ + name: 'observability', + fieldMap: { + ...pickWithPatterns(ecsFieldMap, 'host.name', 'service.name'), + }, +}) +``` + +`fieldMap` is a key-value map of field names and mapping options: + +```ts +{ + '@timestamp': { + type: 'date', + array: false, + required: true, + } +} +``` + +ECS mappings are generated via a script in the rule registry plugin directory. These mappings are available in x-pack/plugins/rule_registry/server/generated/ecs_field_map.ts. + +To pick many fields, you can use `pickWithPatterns`, which supports wildcards with full type support. + +If a registry is created, it will initialise as soon as the core services needed become available. It will create a (versioned) template, alias, and ILM policy, but only if these do not exist yet. + +### Rule registry client + +The rule registry client can either be injected in the executor, or created in the scope of a request. It exposes a `search` method and a `bulkIndex` method. When `search` is called, it first gets all the rules the current user has access to, and adds these ids to the search request that it executes. This means that the user can only see data from rules they have access to. + +Both `search` and `bulkIndex` are fully typed, in the sense that they reflect the mappings defined for the registry. + +### Schema + +The following fields are available in the root rule registry: + +- `@timestamp`: the ISO timestamp of the alert event. For the lifecycle rule type helper, it is always the value of `startedAt` that is injected by the Kibana alerting framework. +- `event.kind`: signal (for the changeable alert document), state (for the state changes of the alert, e.g. when it opens, recovers, or changes in severity), or metric (individual evaluations that might be related to an alert). +- `event.action`: the reason for the event. This might be `open`, `close`, `active`, or `evaluate`. +- `tags`: tags attached to the alert. Right now they are copied over from the rule. +- `rule.id`: the identifier of the rule type, e.g. `apm.transaction_duration` +- `rule.uuid`: the saved objects id of the rule. +- `rule.name`: the name of the rule (as specified by the user). +- `rule.category`: the name of the rule type (as defined by the rule type producer) +- `kibana.rac.producer`: the producer of the rule type. Usually a Kibana plugin. e.g., `APM`. +- `kibana.rac.alert.id`: the id of the alert, that is unique within the context of the rule execution it was created in. E.g., for a rule that monitors latency for all services in all environments, this might be `opbeans-java:production`. +- `kibana.rac.alert.uuid`: the unique identifier for the alert during its lifespan. If an alert recovers (or closes), this identifier is re-generated when it is opened again. +- `kibana.rac.alert.status`: the status of the alert. Can be `open` or `closed`. +- `kibana.rac.alert.start`: the ISO timestamp of the time at which the alert started. +- `kibana.rac.alert.end`: the ISO timestamp of the time at which the alert recovered. +- `kibana.rac.alert.duration.us`: the duration of the alert, in microseconds. This is always the difference between either the current time, or the time when the alert recovered. +- `kibana.rac.alert.severity.level`: the severity of the alert, as a keyword (e.g. critical). +- `kibana.rac.alert.severity.value`: the severity of the alert, as a numerical value, which allows sorting. + +This list is not final - just a start. Field names might change or moved to a scoped registry. If we implement log and sequence based rule types the list of fields will grow. If a rule type needs additional fields, the recommendation would be to have the field in its own registry first (or in its producer’s registry), and if usage is more broadly adopted, it can be moved to the root registry. diff --git a/x-pack/plugins/rule_registry/common/index.ts b/x-pack/plugins/rule_registry/common/index.ts new file mode 100644 index 0000000000000..6cc0ccaa93a6d --- /dev/null +++ b/x-pack/plugins/rule_registry/common/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './types'; diff --git a/x-pack/plugins/rule_registry/common/types.ts b/x-pack/plugins/rule_registry/common/types.ts new file mode 100644 index 0000000000000..d0d15d86a2248 --- /dev/null +++ b/x-pack/plugins/rule_registry/common/types.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export enum AlertSeverityLevel { + warning = 'warning', + critical = 'critical', +} + +const alertSeverityLevelValues = { + [AlertSeverityLevel.warning]: 70, + [AlertSeverityLevel.critical]: 90, +}; + +export function getAlertSeverityLevelValue(level: AlertSeverityLevel) { + return alertSeverityLevelValues[level]; +} diff --git a/x-pack/plugins/rule_registry/jest.config.js b/x-pack/plugins/rule_registry/jest.config.js new file mode 100644 index 0000000000000..df8ac522e4b5d --- /dev/null +++ b/x-pack/plugins/rule_registry/jest.config.js @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +module.exports = { + preset: '@kbn/test', + rootDir: '../../..', + roots: ['/x-pack/plugins/rule_registry'], +}; diff --git a/x-pack/plugins/rule_registry/kibana.json b/x-pack/plugins/rule_registry/kibana.json new file mode 100644 index 0000000000000..dea6ef560cc2d --- /dev/null +++ b/x-pack/plugins/rule_registry/kibana.json @@ -0,0 +1,13 @@ +{ + "id": "ruleRegistry", + "version": "8.0.0", + "kibanaVersion": "kibana", + "configPath": [ + "xpack", + "ruleRegistry" + ], + "requiredPlugins": [ + "alerting" + ], + "server": true +} diff --git a/x-pack/plugins/rule_registry/scripts/generate_ecs_fieldmap/index.js b/x-pack/plugins/rule_registry/scripts/generate_ecs_fieldmap/index.js new file mode 100644 index 0000000000000..6e3a8f7cbe663 --- /dev/null +++ b/x-pack/plugins/rule_registry/scripts/generate_ecs_fieldmap/index.js @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +const path = require('path'); +const fs = require('fs'); +const util = require('util'); +const yaml = require('js-yaml'); +const { exec: execCb } = require('child_process'); +const { mapValues } = require('lodash'); + +const exists = util.promisify(fs.exists); +const readFile = util.promisify(fs.readFile); +const writeFile = util.promisify(fs.writeFile); +const mkdir = util.promisify(fs.mkdir); +const rmdir = util.promisify(fs.rmdir); +const exec = util.promisify(execCb); + +const ecsDir = path.resolve(__dirname, '../../../../../../ecs'); +const ecsTemplateFilename = path.join(ecsDir, 'generated/elasticsearch/7/template.json'); +const flatYamlFilename = path.join(ecsDir, 'generated/ecs/ecs_flat.yml'); + +const outputDir = path.join(__dirname, '../../server/generated'); + +const outputFieldMapFilename = path.join(outputDir, 'ecs_field_map.ts'); +const outputMappingFilename = path.join(outputDir, 'ecs_mappings.json'); + +async function generate() { + const allExists = await Promise.all([exists(ecsDir), exists(ecsTemplateFilename)]); + + if (!allExists.every(Boolean)) { + throw new Error( + `Directory not found: ${ecsDir} - did you checkout elastic/ecs as a peer of this repo?` + ); + } + + const [template, flatYaml] = await Promise.all([ + readFile(ecsTemplateFilename, { encoding: 'utf-8' }).then((str) => JSON.parse(str)), + (async () => yaml.safeLoad(await readFile(flatYamlFilename)))(), + ]); + + const mappings = { + properties: template.mappings.properties, + }; + + const fields = mapValues(flatYaml, (description) => { + return { + type: description.type, + array: description.normalize.includes('array'), + required: !!description.required, + }; + }); + + const hasOutputDir = await exists(outputDir); + + if (hasOutputDir) { + await rmdir(outputDir, { recursive: true }); + } + + await mkdir(outputDir); + + await Promise.all([ + writeFile( + outputFieldMapFilename, + ` + export const ecsFieldMap = ${JSON.stringify(fields, null, 2)} as const + `, + { encoding: 'utf-8' } + ).then(() => { + return exec(`node scripts/eslint --fix ${outputFieldMapFilename}`); + }), + writeFile(outputMappingFilename, JSON.stringify(mappings, null, 2)), + ]); +} + +generate().catch((err) => { + console.log(err); + process.exit(1); +}); diff --git a/x-pack/plugins/rule_registry/server/generated/ecs_field_map.ts b/x-pack/plugins/rule_registry/server/generated/ecs_field_map.ts new file mode 100644 index 0000000000000..cd8865a3f57c2 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/generated/ecs_field_map.ts @@ -0,0 +1,3374 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const ecsFieldMap = { + '@timestamp': { + type: 'date', + array: false, + required: true, + }, + 'agent.build.original': { + type: 'keyword', + array: false, + required: false, + }, + 'agent.ephemeral_id': { + type: 'keyword', + array: false, + required: false, + }, + 'agent.id': { + type: 'keyword', + array: false, + required: false, + }, + 'agent.name': { + type: 'keyword', + array: false, + required: false, + }, + 'agent.type': { + type: 'keyword', + array: false, + required: false, + }, + 'agent.version': { + type: 'keyword', + array: false, + required: false, + }, + 'client.address': { + type: 'keyword', + array: false, + required: false, + }, + 'client.as.number': { + type: 'long', + array: false, + required: false, + }, + 'client.as.organization.name': { + type: 'keyword', + array: false, + required: false, + }, + 'client.bytes': { + type: 'long', + array: false, + required: false, + }, + 'client.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'client.geo.city_name': { + type: 'keyword', + array: false, + required: false, + }, + 'client.geo.continent_name': { + type: 'keyword', + array: false, + required: false, + }, + 'client.geo.country_iso_code': { + type: 'keyword', + array: false, + required: false, + }, + 'client.geo.country_name': { + type: 'keyword', + array: false, + required: false, + }, + 'client.geo.location': { + type: 'geo_point', + array: false, + required: false, + }, + 'client.geo.name': { + type: 'keyword', + array: false, + required: false, + }, + 'client.geo.region_iso_code': { + type: 'keyword', + array: false, + required: false, + }, + 'client.geo.region_name': { + type: 'keyword', + array: false, + required: false, + }, + 'client.ip': { + type: 'ip', + array: false, + required: false, + }, + 'client.mac': { + type: 'keyword', + array: false, + required: false, + }, + 'client.nat.ip': { + type: 'ip', + array: false, + required: false, + }, + 'client.nat.port': { + type: 'long', + array: false, + required: false, + }, + 'client.packets': { + type: 'long', + array: false, + required: false, + }, + 'client.port': { + type: 'long', + array: false, + required: false, + }, + 'client.registered_domain': { + type: 'keyword', + array: false, + required: false, + }, + 'client.subdomain': { + type: 'keyword', + array: false, + required: false, + }, + 'client.top_level_domain': { + type: 'keyword', + array: false, + required: false, + }, + 'client.user.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'client.user.email': { + type: 'keyword', + array: false, + required: false, + }, + 'client.user.full_name': { + type: 'keyword', + array: false, + required: false, + }, + 'client.user.group.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'client.user.group.id': { + type: 'keyword', + array: false, + required: false, + }, + 'client.user.group.name': { + type: 'keyword', + array: false, + required: false, + }, + 'client.user.hash': { + type: 'keyword', + array: false, + required: false, + }, + 'client.user.id': { + type: 'keyword', + array: false, + required: false, + }, + 'client.user.name': { + type: 'keyword', + array: false, + required: false, + }, + 'client.user.roles': { + type: 'keyword', + array: true, + required: false, + }, + 'cloud.account.id': { + type: 'keyword', + array: false, + required: false, + }, + 'cloud.account.name': { + type: 'keyword', + array: false, + required: false, + }, + 'cloud.availability_zone': { + type: 'keyword', + array: false, + required: false, + }, + 'cloud.instance.id': { + type: 'keyword', + array: false, + required: false, + }, + 'cloud.instance.name': { + type: 'keyword', + array: false, + required: false, + }, + 'cloud.machine.type': { + type: 'keyword', + array: false, + required: false, + }, + 'cloud.project.id': { + type: 'keyword', + array: false, + required: false, + }, + 'cloud.project.name': { + type: 'keyword', + array: false, + required: false, + }, + 'cloud.provider': { + type: 'keyword', + array: false, + required: false, + }, + 'cloud.region': { + type: 'keyword', + array: false, + required: false, + }, + 'container.id': { + type: 'keyword', + array: false, + required: false, + }, + 'container.image.name': { + type: 'keyword', + array: false, + required: false, + }, + 'container.image.tag': { + type: 'keyword', + array: true, + required: false, + }, + 'container.labels': { + type: 'object', + array: false, + required: false, + }, + 'container.name': { + type: 'keyword', + array: false, + required: false, + }, + 'container.runtime': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.address': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.as.number': { + type: 'long', + array: false, + required: false, + }, + 'destination.as.organization.name': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.bytes': { + type: 'long', + array: false, + required: false, + }, + 'destination.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.geo.city_name': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.geo.continent_name': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.geo.country_iso_code': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.geo.country_name': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.geo.location': { + type: 'geo_point', + array: false, + required: false, + }, + 'destination.geo.name': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.geo.region_iso_code': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.geo.region_name': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.ip': { + type: 'ip', + array: false, + required: false, + }, + 'destination.mac': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.nat.ip': { + type: 'ip', + array: false, + required: false, + }, + 'destination.nat.port': { + type: 'long', + array: false, + required: false, + }, + 'destination.packets': { + type: 'long', + array: false, + required: false, + }, + 'destination.port': { + type: 'long', + array: false, + required: false, + }, + 'destination.registered_domain': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.subdomain': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.top_level_domain': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.user.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.user.email': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.user.full_name': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.user.group.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.user.group.id': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.user.group.name': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.user.hash': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.user.id': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.user.name': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.user.roles': { + type: 'keyword', + array: true, + required: false, + }, + 'dll.code_signature.exists': { + type: 'boolean', + array: false, + required: false, + }, + 'dll.code_signature.status': { + type: 'keyword', + array: false, + required: false, + }, + 'dll.code_signature.subject_name': { + type: 'keyword', + array: false, + required: false, + }, + 'dll.code_signature.trusted': { + type: 'boolean', + array: false, + required: false, + }, + 'dll.code_signature.valid': { + type: 'boolean', + array: false, + required: false, + }, + 'dll.hash.md5': { + type: 'keyword', + array: false, + required: false, + }, + 'dll.hash.sha1': { + type: 'keyword', + array: false, + required: false, + }, + 'dll.hash.sha256': { + type: 'keyword', + array: false, + required: false, + }, + 'dll.hash.sha512': { + type: 'keyword', + array: false, + required: false, + }, + 'dll.name': { + type: 'keyword', + array: false, + required: false, + }, + 'dll.path': { + type: 'keyword', + array: false, + required: false, + }, + 'dll.pe.architecture': { + type: 'keyword', + array: false, + required: false, + }, + 'dll.pe.company': { + type: 'keyword', + array: false, + required: false, + }, + 'dll.pe.description': { + type: 'keyword', + array: false, + required: false, + }, + 'dll.pe.file_version': { + type: 'keyword', + array: false, + required: false, + }, + 'dll.pe.imphash': { + type: 'keyword', + array: false, + required: false, + }, + 'dll.pe.original_file_name': { + type: 'keyword', + array: false, + required: false, + }, + 'dll.pe.product': { + type: 'keyword', + array: false, + required: false, + }, + 'dns.answers': { + type: 'object', + array: true, + required: false, + }, + 'dns.answers.class': { + type: 'keyword', + array: false, + required: false, + }, + 'dns.answers.data': { + type: 'keyword', + array: false, + required: false, + }, + 'dns.answers.name': { + type: 'keyword', + array: false, + required: false, + }, + 'dns.answers.ttl': { + type: 'long', + array: false, + required: false, + }, + 'dns.answers.type': { + type: 'keyword', + array: false, + required: false, + }, + 'dns.header_flags': { + type: 'keyword', + array: true, + required: false, + }, + 'dns.id': { + type: 'keyword', + array: false, + required: false, + }, + 'dns.op_code': { + type: 'keyword', + array: false, + required: false, + }, + 'dns.question.class': { + type: 'keyword', + array: false, + required: false, + }, + 'dns.question.name': { + type: 'keyword', + array: false, + required: false, + }, + 'dns.question.registered_domain': { + type: 'keyword', + array: false, + required: false, + }, + 'dns.question.subdomain': { + type: 'keyword', + array: false, + required: false, + }, + 'dns.question.top_level_domain': { + type: 'keyword', + array: false, + required: false, + }, + 'dns.question.type': { + type: 'keyword', + array: false, + required: false, + }, + 'dns.resolved_ip': { + type: 'ip', + array: true, + required: false, + }, + 'dns.response_code': { + type: 'keyword', + array: false, + required: false, + }, + 'dns.type': { + type: 'keyword', + array: false, + required: false, + }, + 'ecs.version': { + type: 'keyword', + array: false, + required: true, + }, + 'error.code': { + type: 'keyword', + array: false, + required: false, + }, + 'error.id': { + type: 'keyword', + array: false, + required: false, + }, + 'error.message': { + type: 'text', + array: false, + required: false, + }, + 'error.stack_trace': { + type: 'keyword', + array: false, + required: false, + }, + 'error.type': { + type: 'keyword', + array: false, + required: false, + }, + 'event.action': { + type: 'keyword', + array: false, + required: false, + }, + 'event.category': { + type: 'keyword', + array: true, + required: false, + }, + 'event.code': { + type: 'keyword', + array: false, + required: false, + }, + 'event.created': { + type: 'date', + array: false, + required: false, + }, + 'event.dataset': { + type: 'keyword', + array: false, + required: false, + }, + 'event.duration': { + type: 'long', + array: false, + required: false, + }, + 'event.end': { + type: 'date', + array: false, + required: false, + }, + 'event.hash': { + type: 'keyword', + array: false, + required: false, + }, + 'event.id': { + type: 'keyword', + array: false, + required: false, + }, + 'event.ingested': { + type: 'date', + array: false, + required: false, + }, + 'event.kind': { + type: 'keyword', + array: false, + required: false, + }, + 'event.module': { + type: 'keyword', + array: false, + required: false, + }, + 'event.original': { + type: 'keyword', + array: false, + required: false, + }, + 'event.outcome': { + type: 'keyword', + array: false, + required: false, + }, + 'event.provider': { + type: 'keyword', + array: false, + required: false, + }, + 'event.reason': { + type: 'keyword', + array: false, + required: false, + }, + 'event.reference': { + type: 'keyword', + array: false, + required: false, + }, + 'event.risk_score': { + type: 'float', + array: false, + required: false, + }, + 'event.risk_score_norm': { + type: 'float', + array: false, + required: false, + }, + 'event.sequence': { + type: 'long', + array: false, + required: false, + }, + 'event.severity': { + type: 'long', + array: false, + required: false, + }, + 'event.start': { + type: 'date', + array: false, + required: false, + }, + 'event.timezone': { + type: 'keyword', + array: false, + required: false, + }, + 'event.type': { + type: 'keyword', + array: true, + required: false, + }, + 'event.url': { + type: 'keyword', + array: false, + required: false, + }, + 'file.accessed': { + type: 'date', + array: false, + required: false, + }, + 'file.attributes': { + type: 'keyword', + array: true, + required: false, + }, + 'file.code_signature.exists': { + type: 'boolean', + array: false, + required: false, + }, + 'file.code_signature.status': { + type: 'keyword', + array: false, + required: false, + }, + 'file.code_signature.subject_name': { + type: 'keyword', + array: false, + required: false, + }, + 'file.code_signature.trusted': { + type: 'boolean', + array: false, + required: false, + }, + 'file.code_signature.valid': { + type: 'boolean', + array: false, + required: false, + }, + 'file.created': { + type: 'date', + array: false, + required: false, + }, + 'file.ctime': { + type: 'date', + array: false, + required: false, + }, + 'file.device': { + type: 'keyword', + array: false, + required: false, + }, + 'file.directory': { + type: 'keyword', + array: false, + required: false, + }, + 'file.drive_letter': { + type: 'keyword', + array: false, + required: false, + }, + 'file.extension': { + type: 'keyword', + array: false, + required: false, + }, + 'file.gid': { + type: 'keyword', + array: false, + required: false, + }, + 'file.group': { + type: 'keyword', + array: false, + required: false, + }, + 'file.hash.md5': { + type: 'keyword', + array: false, + required: false, + }, + 'file.hash.sha1': { + type: 'keyword', + array: false, + required: false, + }, + 'file.hash.sha256': { + type: 'keyword', + array: false, + required: false, + }, + 'file.hash.sha512': { + type: 'keyword', + array: false, + required: false, + }, + 'file.inode': { + type: 'keyword', + array: false, + required: false, + }, + 'file.mime_type': { + type: 'keyword', + array: false, + required: false, + }, + 'file.mode': { + type: 'keyword', + array: false, + required: false, + }, + 'file.mtime': { + type: 'date', + array: false, + required: false, + }, + 'file.name': { + type: 'keyword', + array: false, + required: false, + }, + 'file.owner': { + type: 'keyword', + array: false, + required: false, + }, + 'file.path': { + type: 'keyword', + array: false, + required: false, + }, + 'file.pe.architecture': { + type: 'keyword', + array: false, + required: false, + }, + 'file.pe.company': { + type: 'keyword', + array: false, + required: false, + }, + 'file.pe.description': { + type: 'keyword', + array: false, + required: false, + }, + 'file.pe.file_version': { + type: 'keyword', + array: false, + required: false, + }, + 'file.pe.imphash': { + type: 'keyword', + array: false, + required: false, + }, + 'file.pe.original_file_name': { + type: 'keyword', + array: false, + required: false, + }, + 'file.pe.product': { + type: 'keyword', + array: false, + required: false, + }, + 'file.size': { + type: 'long', + array: false, + required: false, + }, + 'file.target_path': { + type: 'keyword', + array: false, + required: false, + }, + 'file.type': { + type: 'keyword', + array: false, + required: false, + }, + 'file.uid': { + type: 'keyword', + array: false, + required: false, + }, + 'file.x509.alternative_names': { + type: 'keyword', + array: true, + required: false, + }, + 'file.x509.issuer.common_name': { + type: 'keyword', + array: true, + required: false, + }, + 'file.x509.issuer.country': { + type: 'keyword', + array: true, + required: false, + }, + 'file.x509.issuer.distinguished_name': { + type: 'keyword', + array: false, + required: false, + }, + 'file.x509.issuer.locality': { + type: 'keyword', + array: true, + required: false, + }, + 'file.x509.issuer.organization': { + type: 'keyword', + array: true, + required: false, + }, + 'file.x509.issuer.organizational_unit': { + type: 'keyword', + array: true, + required: false, + }, + 'file.x509.issuer.state_or_province': { + type: 'keyword', + array: true, + required: false, + }, + 'file.x509.not_after': { + type: 'date', + array: false, + required: false, + }, + 'file.x509.not_before': { + type: 'date', + array: false, + required: false, + }, + 'file.x509.public_key_algorithm': { + type: 'keyword', + array: false, + required: false, + }, + 'file.x509.public_key_curve': { + type: 'keyword', + array: false, + required: false, + }, + 'file.x509.public_key_exponent': { + type: 'long', + array: false, + required: false, + }, + 'file.x509.public_key_size': { + type: 'long', + array: false, + required: false, + }, + 'file.x509.serial_number': { + type: 'keyword', + array: false, + required: false, + }, + 'file.x509.signature_algorithm': { + type: 'keyword', + array: false, + required: false, + }, + 'file.x509.subject.common_name': { + type: 'keyword', + array: true, + required: false, + }, + 'file.x509.subject.country': { + type: 'keyword', + array: true, + required: false, + }, + 'file.x509.subject.distinguished_name': { + type: 'keyword', + array: false, + required: false, + }, + 'file.x509.subject.locality': { + type: 'keyword', + array: true, + required: false, + }, + 'file.x509.subject.organization': { + type: 'keyword', + array: true, + required: false, + }, + 'file.x509.subject.organizational_unit': { + type: 'keyword', + array: true, + required: false, + }, + 'file.x509.subject.state_or_province': { + type: 'keyword', + array: true, + required: false, + }, + 'file.x509.version_number': { + type: 'keyword', + array: false, + required: false, + }, + 'group.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'group.id': { + type: 'keyword', + array: false, + required: false, + }, + 'group.name': { + type: 'keyword', + array: false, + required: false, + }, + 'host.architecture': { + type: 'keyword', + array: false, + required: false, + }, + 'host.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'host.geo.city_name': { + type: 'keyword', + array: false, + required: false, + }, + 'host.geo.continent_name': { + type: 'keyword', + array: false, + required: false, + }, + 'host.geo.country_iso_code': { + type: 'keyword', + array: false, + required: false, + }, + 'host.geo.country_name': { + type: 'keyword', + array: false, + required: false, + }, + 'host.geo.location': { + type: 'geo_point', + array: false, + required: false, + }, + 'host.geo.name': { + type: 'keyword', + array: false, + required: false, + }, + 'host.geo.region_iso_code': { + type: 'keyword', + array: false, + required: false, + }, + 'host.geo.region_name': { + type: 'keyword', + array: false, + required: false, + }, + 'host.hostname': { + type: 'keyword', + array: false, + required: false, + }, + 'host.id': { + type: 'keyword', + array: false, + required: false, + }, + 'host.ip': { + type: 'ip', + array: true, + required: false, + }, + 'host.mac': { + type: 'keyword', + array: true, + required: false, + }, + 'host.name': { + type: 'keyword', + array: false, + required: false, + }, + 'host.os.family': { + type: 'keyword', + array: false, + required: false, + }, + 'host.os.full': { + type: 'keyword', + array: false, + required: false, + }, + 'host.os.kernel': { + type: 'keyword', + array: false, + required: false, + }, + 'host.os.name': { + type: 'keyword', + array: false, + required: false, + }, + 'host.os.platform': { + type: 'keyword', + array: false, + required: false, + }, + 'host.os.type': { + type: 'keyword', + array: false, + required: false, + }, + 'host.os.version': { + type: 'keyword', + array: false, + required: false, + }, + 'host.type': { + type: 'keyword', + array: false, + required: false, + }, + 'host.uptime': { + type: 'long', + array: false, + required: false, + }, + 'host.user.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'host.user.email': { + type: 'keyword', + array: false, + required: false, + }, + 'host.user.full_name': { + type: 'keyword', + array: false, + required: false, + }, + 'host.user.group.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'host.user.group.id': { + type: 'keyword', + array: false, + required: false, + }, + 'host.user.group.name': { + type: 'keyword', + array: false, + required: false, + }, + 'host.user.hash': { + type: 'keyword', + array: false, + required: false, + }, + 'host.user.id': { + type: 'keyword', + array: false, + required: false, + }, + 'host.user.name': { + type: 'keyword', + array: false, + required: false, + }, + 'host.user.roles': { + type: 'keyword', + array: true, + required: false, + }, + 'http.request.body.bytes': { + type: 'long', + array: false, + required: false, + }, + 'http.request.body.content': { + type: 'keyword', + array: false, + required: false, + }, + 'http.request.bytes': { + type: 'long', + array: false, + required: false, + }, + 'http.request.method': { + type: 'keyword', + array: false, + required: false, + }, + 'http.request.mime_type': { + type: 'keyword', + array: false, + required: false, + }, + 'http.request.referrer': { + type: 'keyword', + array: false, + required: false, + }, + 'http.response.body.bytes': { + type: 'long', + array: false, + required: false, + }, + 'http.response.body.content': { + type: 'keyword', + array: false, + required: false, + }, + 'http.response.bytes': { + type: 'long', + array: false, + required: false, + }, + 'http.response.mime_type': { + type: 'keyword', + array: false, + required: false, + }, + 'http.response.status_code': { + type: 'long', + array: false, + required: false, + }, + 'http.version': { + type: 'keyword', + array: false, + required: false, + }, + labels: { + type: 'object', + array: false, + required: false, + }, + 'log.file.path': { + type: 'keyword', + array: false, + required: false, + }, + 'log.level': { + type: 'keyword', + array: false, + required: false, + }, + 'log.logger': { + type: 'keyword', + array: false, + required: false, + }, + 'log.origin.file.line': { + type: 'integer', + array: false, + required: false, + }, + 'log.origin.file.name': { + type: 'keyword', + array: false, + required: false, + }, + 'log.origin.function': { + type: 'keyword', + array: false, + required: false, + }, + 'log.original': { + type: 'keyword', + array: false, + required: false, + }, + 'log.syslog': { + type: 'object', + array: false, + required: false, + }, + 'log.syslog.facility.code': { + type: 'long', + array: false, + required: false, + }, + 'log.syslog.facility.name': { + type: 'keyword', + array: false, + required: false, + }, + 'log.syslog.priority': { + type: 'long', + array: false, + required: false, + }, + 'log.syslog.severity.code': { + type: 'long', + array: false, + required: false, + }, + 'log.syslog.severity.name': { + type: 'keyword', + array: false, + required: false, + }, + message: { + type: 'text', + array: false, + required: false, + }, + 'network.application': { + type: 'keyword', + array: false, + required: false, + }, + 'network.bytes': { + type: 'long', + array: false, + required: false, + }, + 'network.community_id': { + type: 'keyword', + array: false, + required: false, + }, + 'network.direction': { + type: 'keyword', + array: false, + required: false, + }, + 'network.forwarded_ip': { + type: 'ip', + array: false, + required: false, + }, + 'network.iana_number': { + type: 'keyword', + array: false, + required: false, + }, + 'network.inner': { + type: 'object', + array: false, + required: false, + }, + 'network.inner.vlan.id': { + type: 'keyword', + array: false, + required: false, + }, + 'network.inner.vlan.name': { + type: 'keyword', + array: false, + required: false, + }, + 'network.name': { + type: 'keyword', + array: false, + required: false, + }, + 'network.packets': { + type: 'long', + array: false, + required: false, + }, + 'network.protocol': { + type: 'keyword', + array: false, + required: false, + }, + 'network.transport': { + type: 'keyword', + array: false, + required: false, + }, + 'network.type': { + type: 'keyword', + array: false, + required: false, + }, + 'network.vlan.id': { + type: 'keyword', + array: false, + required: false, + }, + 'network.vlan.name': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.egress': { + type: 'object', + array: false, + required: false, + }, + 'observer.egress.interface.alias': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.egress.interface.id': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.egress.interface.name': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.egress.vlan.id': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.egress.vlan.name': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.egress.zone': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.geo.city_name': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.geo.continent_name': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.geo.country_iso_code': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.geo.country_name': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.geo.location': { + type: 'geo_point', + array: false, + required: false, + }, + 'observer.geo.name': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.geo.region_iso_code': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.geo.region_name': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.hostname': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.ingress': { + type: 'object', + array: false, + required: false, + }, + 'observer.ingress.interface.alias': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.ingress.interface.id': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.ingress.interface.name': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.ingress.vlan.id': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.ingress.vlan.name': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.ingress.zone': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.ip': { + type: 'ip', + array: true, + required: false, + }, + 'observer.mac': { + type: 'keyword', + array: true, + required: false, + }, + 'observer.name': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.os.family': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.os.full': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.os.kernel': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.os.name': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.os.platform': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.os.type': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.os.version': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.product': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.serial_number': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.type': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.vendor': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.version': { + type: 'keyword', + array: false, + required: false, + }, + 'organization.id': { + type: 'keyword', + array: false, + required: false, + }, + 'organization.name': { + type: 'keyword', + array: false, + required: false, + }, + 'package.architecture': { + type: 'keyword', + array: false, + required: false, + }, + 'package.build_version': { + type: 'keyword', + array: false, + required: false, + }, + 'package.checksum': { + type: 'keyword', + array: false, + required: false, + }, + 'package.description': { + type: 'keyword', + array: false, + required: false, + }, + 'package.install_scope': { + type: 'keyword', + array: false, + required: false, + }, + 'package.installed': { + type: 'date', + array: false, + required: false, + }, + 'package.license': { + type: 'keyword', + array: false, + required: false, + }, + 'package.name': { + type: 'keyword', + array: false, + required: false, + }, + 'package.path': { + type: 'keyword', + array: false, + required: false, + }, + 'package.reference': { + type: 'keyword', + array: false, + required: false, + }, + 'package.size': { + type: 'long', + array: false, + required: false, + }, + 'package.type': { + type: 'keyword', + array: false, + required: false, + }, + 'package.version': { + type: 'keyword', + array: false, + required: false, + }, + 'process.args': { + type: 'keyword', + array: true, + required: false, + }, + 'process.args_count': { + type: 'long', + array: false, + required: false, + }, + 'process.code_signature.exists': { + type: 'boolean', + array: false, + required: false, + }, + 'process.code_signature.status': { + type: 'keyword', + array: false, + required: false, + }, + 'process.code_signature.subject_name': { + type: 'keyword', + array: false, + required: false, + }, + 'process.code_signature.trusted': { + type: 'boolean', + array: false, + required: false, + }, + 'process.code_signature.valid': { + type: 'boolean', + array: false, + required: false, + }, + 'process.command_line': { + type: 'keyword', + array: false, + required: false, + }, + 'process.entity_id': { + type: 'keyword', + array: false, + required: false, + }, + 'process.executable': { + type: 'keyword', + array: false, + required: false, + }, + 'process.exit_code': { + type: 'long', + array: false, + required: false, + }, + 'process.hash.md5': { + type: 'keyword', + array: false, + required: false, + }, + 'process.hash.sha1': { + type: 'keyword', + array: false, + required: false, + }, + 'process.hash.sha256': { + type: 'keyword', + array: false, + required: false, + }, + 'process.hash.sha512': { + type: 'keyword', + array: false, + required: false, + }, + 'process.name': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.args': { + type: 'keyword', + array: true, + required: false, + }, + 'process.parent.args_count': { + type: 'long', + array: false, + required: false, + }, + 'process.parent.code_signature.exists': { + type: 'boolean', + array: false, + required: false, + }, + 'process.parent.code_signature.status': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.code_signature.subject_name': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.code_signature.trusted': { + type: 'boolean', + array: false, + required: false, + }, + 'process.parent.code_signature.valid': { + type: 'boolean', + array: false, + required: false, + }, + 'process.parent.command_line': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.entity_id': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.executable': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.exit_code': { + type: 'long', + array: false, + required: false, + }, + 'process.parent.hash.md5': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.hash.sha1': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.hash.sha256': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.hash.sha512': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.name': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.pe.architecture': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.pe.company': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.pe.description': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.pe.file_version': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.pe.imphash': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.pe.original_file_name': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.pe.product': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.pgid': { + type: 'long', + array: false, + required: false, + }, + 'process.parent.pid': { + type: 'long', + array: false, + required: false, + }, + 'process.parent.ppid': { + type: 'long', + array: false, + required: false, + }, + 'process.parent.start': { + type: 'date', + array: false, + required: false, + }, + 'process.parent.thread.id': { + type: 'long', + array: false, + required: false, + }, + 'process.parent.thread.name': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.title': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.uptime': { + type: 'long', + array: false, + required: false, + }, + 'process.parent.working_directory': { + type: 'keyword', + array: false, + required: false, + }, + 'process.pe.architecture': { + type: 'keyword', + array: false, + required: false, + }, + 'process.pe.company': { + type: 'keyword', + array: false, + required: false, + }, + 'process.pe.description': { + type: 'keyword', + array: false, + required: false, + }, + 'process.pe.file_version': { + type: 'keyword', + array: false, + required: false, + }, + 'process.pe.imphash': { + type: 'keyword', + array: false, + required: false, + }, + 'process.pe.original_file_name': { + type: 'keyword', + array: false, + required: false, + }, + 'process.pe.product': { + type: 'keyword', + array: false, + required: false, + }, + 'process.pgid': { + type: 'long', + array: false, + required: false, + }, + 'process.pid': { + type: 'long', + array: false, + required: false, + }, + 'process.ppid': { + type: 'long', + array: false, + required: false, + }, + 'process.start': { + type: 'date', + array: false, + required: false, + }, + 'process.thread.id': { + type: 'long', + array: false, + required: false, + }, + 'process.thread.name': { + type: 'keyword', + array: false, + required: false, + }, + 'process.title': { + type: 'keyword', + array: false, + required: false, + }, + 'process.uptime': { + type: 'long', + array: false, + required: false, + }, + 'process.working_directory': { + type: 'keyword', + array: false, + required: false, + }, + 'registry.data.bytes': { + type: 'keyword', + array: false, + required: false, + }, + 'registry.data.strings': { + type: 'keyword', + array: true, + required: false, + }, + 'registry.data.type': { + type: 'keyword', + array: false, + required: false, + }, + 'registry.hive': { + type: 'keyword', + array: false, + required: false, + }, + 'registry.key': { + type: 'keyword', + array: false, + required: false, + }, + 'registry.path': { + type: 'keyword', + array: false, + required: false, + }, + 'registry.value': { + type: 'keyword', + array: false, + required: false, + }, + 'related.hash': { + type: 'keyword', + array: true, + required: false, + }, + 'related.hosts': { + type: 'keyword', + array: true, + required: false, + }, + 'related.ip': { + type: 'ip', + array: true, + required: false, + }, + 'related.user': { + type: 'keyword', + array: true, + required: false, + }, + 'rule.author': { + type: 'keyword', + array: true, + required: false, + }, + 'rule.category': { + type: 'keyword', + array: false, + required: false, + }, + 'rule.description': { + type: 'keyword', + array: false, + required: false, + }, + 'rule.id': { + type: 'keyword', + array: false, + required: false, + }, + 'rule.license': { + type: 'keyword', + array: false, + required: false, + }, + 'rule.name': { + type: 'keyword', + array: false, + required: false, + }, + 'rule.reference': { + type: 'keyword', + array: false, + required: false, + }, + 'rule.ruleset': { + type: 'keyword', + array: false, + required: false, + }, + 'rule.uuid': { + type: 'keyword', + array: false, + required: false, + }, + 'rule.version': { + type: 'keyword', + array: false, + required: false, + }, + 'server.address': { + type: 'keyword', + array: false, + required: false, + }, + 'server.as.number': { + type: 'long', + array: false, + required: false, + }, + 'server.as.organization.name': { + type: 'keyword', + array: false, + required: false, + }, + 'server.bytes': { + type: 'long', + array: false, + required: false, + }, + 'server.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'server.geo.city_name': { + type: 'keyword', + array: false, + required: false, + }, + 'server.geo.continent_name': { + type: 'keyword', + array: false, + required: false, + }, + 'server.geo.country_iso_code': { + type: 'keyword', + array: false, + required: false, + }, + 'server.geo.country_name': { + type: 'keyword', + array: false, + required: false, + }, + 'server.geo.location': { + type: 'geo_point', + array: false, + required: false, + }, + 'server.geo.name': { + type: 'keyword', + array: false, + required: false, + }, + 'server.geo.region_iso_code': { + type: 'keyword', + array: false, + required: false, + }, + 'server.geo.region_name': { + type: 'keyword', + array: false, + required: false, + }, + 'server.ip': { + type: 'ip', + array: false, + required: false, + }, + 'server.mac': { + type: 'keyword', + array: false, + required: false, + }, + 'server.nat.ip': { + type: 'ip', + array: false, + required: false, + }, + 'server.nat.port': { + type: 'long', + array: false, + required: false, + }, + 'server.packets': { + type: 'long', + array: false, + required: false, + }, + 'server.port': { + type: 'long', + array: false, + required: false, + }, + 'server.registered_domain': { + type: 'keyword', + array: false, + required: false, + }, + 'server.subdomain': { + type: 'keyword', + array: false, + required: false, + }, + 'server.top_level_domain': { + type: 'keyword', + array: false, + required: false, + }, + 'server.user.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'server.user.email': { + type: 'keyword', + array: false, + required: false, + }, + 'server.user.full_name': { + type: 'keyword', + array: false, + required: false, + }, + 'server.user.group.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'server.user.group.id': { + type: 'keyword', + array: false, + required: false, + }, + 'server.user.group.name': { + type: 'keyword', + array: false, + required: false, + }, + 'server.user.hash': { + type: 'keyword', + array: false, + required: false, + }, + 'server.user.id': { + type: 'keyword', + array: false, + required: false, + }, + 'server.user.name': { + type: 'keyword', + array: false, + required: false, + }, + 'server.user.roles': { + type: 'keyword', + array: true, + required: false, + }, + 'service.ephemeral_id': { + type: 'keyword', + array: false, + required: false, + }, + 'service.id': { + type: 'keyword', + array: false, + required: false, + }, + 'service.name': { + type: 'keyword', + array: false, + required: false, + }, + 'service.node.name': { + type: 'keyword', + array: false, + required: false, + }, + 'service.state': { + type: 'keyword', + array: false, + required: false, + }, + 'service.type': { + type: 'keyword', + array: false, + required: false, + }, + 'service.version': { + type: 'keyword', + array: false, + required: false, + }, + 'source.address': { + type: 'keyword', + array: false, + required: false, + }, + 'source.as.number': { + type: 'long', + array: false, + required: false, + }, + 'source.as.organization.name': { + type: 'keyword', + array: false, + required: false, + }, + 'source.bytes': { + type: 'long', + array: false, + required: false, + }, + 'source.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'source.geo.city_name': { + type: 'keyword', + array: false, + required: false, + }, + 'source.geo.continent_name': { + type: 'keyword', + array: false, + required: false, + }, + 'source.geo.country_iso_code': { + type: 'keyword', + array: false, + required: false, + }, + 'source.geo.country_name': { + type: 'keyword', + array: false, + required: false, + }, + 'source.geo.location': { + type: 'geo_point', + array: false, + required: false, + }, + 'source.geo.name': { + type: 'keyword', + array: false, + required: false, + }, + 'source.geo.region_iso_code': { + type: 'keyword', + array: false, + required: false, + }, + 'source.geo.region_name': { + type: 'keyword', + array: false, + required: false, + }, + 'source.ip': { + type: 'ip', + array: false, + required: false, + }, + 'source.mac': { + type: 'keyword', + array: false, + required: false, + }, + 'source.nat.ip': { + type: 'ip', + array: false, + required: false, + }, + 'source.nat.port': { + type: 'long', + array: false, + required: false, + }, + 'source.packets': { + type: 'long', + array: false, + required: false, + }, + 'source.port': { + type: 'long', + array: false, + required: false, + }, + 'source.registered_domain': { + type: 'keyword', + array: false, + required: false, + }, + 'source.subdomain': { + type: 'keyword', + array: false, + required: false, + }, + 'source.top_level_domain': { + type: 'keyword', + array: false, + required: false, + }, + 'source.user.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'source.user.email': { + type: 'keyword', + array: false, + required: false, + }, + 'source.user.full_name': { + type: 'keyword', + array: false, + required: false, + }, + 'source.user.group.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'source.user.group.id': { + type: 'keyword', + array: false, + required: false, + }, + 'source.user.group.name': { + type: 'keyword', + array: false, + required: false, + }, + 'source.user.hash': { + type: 'keyword', + array: false, + required: false, + }, + 'source.user.id': { + type: 'keyword', + array: false, + required: false, + }, + 'source.user.name': { + type: 'keyword', + array: false, + required: false, + }, + 'source.user.roles': { + type: 'keyword', + array: true, + required: false, + }, + 'span.id': { + type: 'keyword', + array: false, + required: false, + }, + tags: { + type: 'keyword', + array: true, + required: false, + }, + 'threat.framework': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.tactic.id': { + type: 'keyword', + array: true, + required: false, + }, + 'threat.tactic.name': { + type: 'keyword', + array: true, + required: false, + }, + 'threat.tactic.reference': { + type: 'keyword', + array: true, + required: false, + }, + 'threat.technique.id': { + type: 'keyword', + array: true, + required: false, + }, + 'threat.technique.name': { + type: 'keyword', + array: true, + required: false, + }, + 'threat.technique.reference': { + type: 'keyword', + array: true, + required: false, + }, + 'threat.technique.subtechnique.id': { + type: 'keyword', + array: true, + required: false, + }, + 'threat.technique.subtechnique.name': { + type: 'keyword', + array: true, + required: false, + }, + 'threat.technique.subtechnique.reference': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.cipher': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.client.certificate': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.client.certificate_chain': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.client.hash.md5': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.client.hash.sha1': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.client.hash.sha256': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.client.issuer': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.client.ja3': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.client.not_after': { + type: 'date', + array: false, + required: false, + }, + 'tls.client.not_before': { + type: 'date', + array: false, + required: false, + }, + 'tls.client.server_name': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.client.subject': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.client.supported_ciphers': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.client.x509.alternative_names': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.client.x509.issuer.common_name': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.client.x509.issuer.country': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.client.x509.issuer.distinguished_name': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.client.x509.issuer.locality': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.client.x509.issuer.organization': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.client.x509.issuer.organizational_unit': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.client.x509.issuer.state_or_province': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.client.x509.not_after': { + type: 'date', + array: false, + required: false, + }, + 'tls.client.x509.not_before': { + type: 'date', + array: false, + required: false, + }, + 'tls.client.x509.public_key_algorithm': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.client.x509.public_key_curve': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.client.x509.public_key_exponent': { + type: 'long', + array: false, + required: false, + }, + 'tls.client.x509.public_key_size': { + type: 'long', + array: false, + required: false, + }, + 'tls.client.x509.serial_number': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.client.x509.signature_algorithm': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.client.x509.subject.common_name': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.client.x509.subject.country': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.client.x509.subject.distinguished_name': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.client.x509.subject.locality': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.client.x509.subject.organization': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.client.x509.subject.organizational_unit': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.client.x509.subject.state_or_province': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.client.x509.version_number': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.curve': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.established': { + type: 'boolean', + array: false, + required: false, + }, + 'tls.next_protocol': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.resumed': { + type: 'boolean', + array: false, + required: false, + }, + 'tls.server.certificate': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.server.certificate_chain': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.server.hash.md5': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.server.hash.sha1': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.server.hash.sha256': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.server.issuer': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.server.ja3s': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.server.not_after': { + type: 'date', + array: false, + required: false, + }, + 'tls.server.not_before': { + type: 'date', + array: false, + required: false, + }, + 'tls.server.subject': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.server.x509.alternative_names': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.server.x509.issuer.common_name': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.server.x509.issuer.country': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.server.x509.issuer.distinguished_name': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.server.x509.issuer.locality': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.server.x509.issuer.organization': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.server.x509.issuer.organizational_unit': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.server.x509.issuer.state_or_province': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.server.x509.not_after': { + type: 'date', + array: false, + required: false, + }, + 'tls.server.x509.not_before': { + type: 'date', + array: false, + required: false, + }, + 'tls.server.x509.public_key_algorithm': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.server.x509.public_key_curve': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.server.x509.public_key_exponent': { + type: 'long', + array: false, + required: false, + }, + 'tls.server.x509.public_key_size': { + type: 'long', + array: false, + required: false, + }, + 'tls.server.x509.serial_number': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.server.x509.signature_algorithm': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.server.x509.subject.common_name': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.server.x509.subject.country': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.server.x509.subject.distinguished_name': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.server.x509.subject.locality': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.server.x509.subject.organization': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.server.x509.subject.organizational_unit': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.server.x509.subject.state_or_province': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.server.x509.version_number': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.version': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.version_protocol': { + type: 'keyword', + array: false, + required: false, + }, + 'trace.id': { + type: 'keyword', + array: false, + required: false, + }, + 'transaction.id': { + type: 'keyword', + array: false, + required: false, + }, + 'url.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'url.extension': { + type: 'keyword', + array: false, + required: false, + }, + 'url.fragment': { + type: 'keyword', + array: false, + required: false, + }, + 'url.full': { + type: 'keyword', + array: false, + required: false, + }, + 'url.original': { + type: 'keyword', + array: false, + required: false, + }, + 'url.password': { + type: 'keyword', + array: false, + required: false, + }, + 'url.path': { + type: 'keyword', + array: false, + required: false, + }, + 'url.port': { + type: 'long', + array: false, + required: false, + }, + 'url.query': { + type: 'keyword', + array: false, + required: false, + }, + 'url.registered_domain': { + type: 'keyword', + array: false, + required: false, + }, + 'url.scheme': { + type: 'keyword', + array: false, + required: false, + }, + 'url.subdomain': { + type: 'keyword', + array: false, + required: false, + }, + 'url.top_level_domain': { + type: 'keyword', + array: false, + required: false, + }, + 'url.username': { + type: 'keyword', + array: false, + required: false, + }, + 'user.changes.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'user.changes.email': { + type: 'keyword', + array: false, + required: false, + }, + 'user.changes.full_name': { + type: 'keyword', + array: false, + required: false, + }, + 'user.changes.group.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'user.changes.group.id': { + type: 'keyword', + array: false, + required: false, + }, + 'user.changes.group.name': { + type: 'keyword', + array: false, + required: false, + }, + 'user.changes.hash': { + type: 'keyword', + array: false, + required: false, + }, + 'user.changes.id': { + type: 'keyword', + array: false, + required: false, + }, + 'user.changes.name': { + type: 'keyword', + array: false, + required: false, + }, + 'user.changes.roles': { + type: 'keyword', + array: true, + required: false, + }, + 'user.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'user.effective.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'user.effective.email': { + type: 'keyword', + array: false, + required: false, + }, + 'user.effective.full_name': { + type: 'keyword', + array: false, + required: false, + }, + 'user.effective.group.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'user.effective.group.id': { + type: 'keyword', + array: false, + required: false, + }, + 'user.effective.group.name': { + type: 'keyword', + array: false, + required: false, + }, + 'user.effective.hash': { + type: 'keyword', + array: false, + required: false, + }, + 'user.effective.id': { + type: 'keyword', + array: false, + required: false, + }, + 'user.effective.name': { + type: 'keyword', + array: false, + required: false, + }, + 'user.effective.roles': { + type: 'keyword', + array: true, + required: false, + }, + 'user.email': { + type: 'keyword', + array: false, + required: false, + }, + 'user.full_name': { + type: 'keyword', + array: false, + required: false, + }, + 'user.group.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'user.group.id': { + type: 'keyword', + array: false, + required: false, + }, + 'user.group.name': { + type: 'keyword', + array: false, + required: false, + }, + 'user.hash': { + type: 'keyword', + array: false, + required: false, + }, + 'user.id': { + type: 'keyword', + array: false, + required: false, + }, + 'user.name': { + type: 'keyword', + array: false, + required: false, + }, + 'user.roles': { + type: 'keyword', + array: true, + required: false, + }, + 'user.target.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'user.target.email': { + type: 'keyword', + array: false, + required: false, + }, + 'user.target.full_name': { + type: 'keyword', + array: false, + required: false, + }, + 'user.target.group.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'user.target.group.id': { + type: 'keyword', + array: false, + required: false, + }, + 'user.target.group.name': { + type: 'keyword', + array: false, + required: false, + }, + 'user.target.hash': { + type: 'keyword', + array: false, + required: false, + }, + 'user.target.id': { + type: 'keyword', + array: false, + required: false, + }, + 'user.target.name': { + type: 'keyword', + array: false, + required: false, + }, + 'user.target.roles': { + type: 'keyword', + array: true, + required: false, + }, + 'user_agent.device.name': { + type: 'keyword', + array: false, + required: false, + }, + 'user_agent.name': { + type: 'keyword', + array: false, + required: false, + }, + 'user_agent.original': { + type: 'keyword', + array: false, + required: false, + }, + 'user_agent.os.family': { + type: 'keyword', + array: false, + required: false, + }, + 'user_agent.os.full': { + type: 'keyword', + array: false, + required: false, + }, + 'user_agent.os.kernel': { + type: 'keyword', + array: false, + required: false, + }, + 'user_agent.os.name': { + type: 'keyword', + array: false, + required: false, + }, + 'user_agent.os.platform': { + type: 'keyword', + array: false, + required: false, + }, + 'user_agent.os.type': { + type: 'keyword', + array: false, + required: false, + }, + 'user_agent.os.version': { + type: 'keyword', + array: false, + required: false, + }, + 'user_agent.version': { + type: 'keyword', + array: false, + required: false, + }, + 'vulnerability.category': { + type: 'keyword', + array: true, + required: false, + }, + 'vulnerability.classification': { + type: 'keyword', + array: false, + required: false, + }, + 'vulnerability.description': { + type: 'keyword', + array: false, + required: false, + }, + 'vulnerability.enumeration': { + type: 'keyword', + array: false, + required: false, + }, + 'vulnerability.id': { + type: 'keyword', + array: false, + required: false, + }, + 'vulnerability.reference': { + type: 'keyword', + array: false, + required: false, + }, + 'vulnerability.report_id': { + type: 'keyword', + array: false, + required: false, + }, + 'vulnerability.scanner.vendor': { + type: 'keyword', + array: false, + required: false, + }, + 'vulnerability.score.base': { + type: 'float', + array: false, + required: false, + }, + 'vulnerability.score.environmental': { + type: 'float', + array: false, + required: false, + }, + 'vulnerability.score.temporal': { + type: 'float', + array: false, + required: false, + }, + 'vulnerability.score.version': { + type: 'keyword', + array: false, + required: false, + }, + 'vulnerability.severity': { + type: 'keyword', + array: false, + required: false, + }, +} as const; diff --git a/x-pack/plugins/rule_registry/server/generated/ecs_mappings.json b/x-pack/plugins/rule_registry/server/generated/ecs_mappings.json new file mode 100644 index 0000000000000..f7cbfc3dfaae3 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/generated/ecs_mappings.json @@ -0,0 +1,3416 @@ +{ + "properties": { + "@timestamp": { + "type": "date" + }, + "agent": { + "properties": { + "build": { + "properties": { + "original": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ephemeral_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "client": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "as": { + "properties": { + "number": { + "type": "long" + }, + "organization": { + "properties": { + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "bytes": { + "type": "long" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "nat": { + "properties": { + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + } + } + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + }, + "registered_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "subdomain": { + "ignore_above": 1024, + "type": "keyword" + }, + "top_level_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "roles": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "cloud": { + "properties": { + "account": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "availability_zone": { + "ignore_above": 1024, + "type": "keyword" + }, + "instance": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "machine": { + "properties": { + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "project": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "provider": { + "ignore_above": 1024, + "type": "keyword" + }, + "region": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "container": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "image": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "tag": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "labels": { + "type": "object" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "runtime": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "destination": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "as": { + "properties": { + "number": { + "type": "long" + }, + "organization": { + "properties": { + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "bytes": { + "type": "long" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "nat": { + "properties": { + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + } + } + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + }, + "registered_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "subdomain": { + "ignore_above": 1024, + "type": "keyword" + }, + "top_level_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "roles": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "dll": { + "properties": { + "code_signature": { + "properties": { + "exists": { + "type": "boolean" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "trusted": { + "type": "boolean" + }, + "valid": { + "type": "boolean" + } + } + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha512": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "pe": { + "properties": { + "architecture": { + "ignore_above": 1024, + "type": "keyword" + }, + "company": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "file_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "imphash": { + "ignore_above": 1024, + "type": "keyword" + }, + "original_file_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "product": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "dns": { + "properties": { + "answers": { + "properties": { + "class": { + "ignore_above": 1024, + "type": "keyword" + }, + "data": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "ttl": { + "type": "long" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + }, + "type": "object" + }, + "header_flags": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "op_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "question": { + "properties": { + "class": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "registered_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "subdomain": { + "ignore_above": 1024, + "type": "keyword" + }, + "top_level_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "resolved_ip": { + "type": "ip" + }, + "response_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ecs": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "error": { + "properties": { + "code": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "norms": false, + "type": "text" + }, + "stack_trace": { + "doc_values": false, + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "index": false, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "event": { + "properties": { + "action": { + "ignore_above": 1024, + "type": "keyword" + }, + "category": { + "ignore_above": 1024, + "type": "keyword" + }, + "code": { + "ignore_above": 1024, + "type": "keyword" + }, + "created": { + "type": "date" + }, + "dataset": { + "ignore_above": 1024, + "type": "keyword" + }, + "duration": { + "type": "long" + }, + "end": { + "type": "date" + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "ingested": { + "type": "date" + }, + "kind": { + "ignore_above": 1024, + "type": "keyword" + }, + "module": { + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "doc_values": false, + "ignore_above": 1024, + "index": false, + "type": "keyword" + }, + "outcome": { + "ignore_above": 1024, + "type": "keyword" + }, + "provider": { + "ignore_above": 1024, + "type": "keyword" + }, + "reason": { + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + }, + "risk_score": { + "type": "float" + }, + "risk_score_norm": { + "type": "float" + }, + "sequence": { + "type": "long" + }, + "severity": { + "type": "long" + }, + "start": { + "type": "date" + }, + "timezone": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "url": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "file": { + "properties": { + "accessed": { + "type": "date" + }, + "attributes": { + "ignore_above": 1024, + "type": "keyword" + }, + "code_signature": { + "properties": { + "exists": { + "type": "boolean" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "trusted": { + "type": "boolean" + }, + "valid": { + "type": "boolean" + } + } + }, + "created": { + "type": "date" + }, + "ctime": { + "type": "date" + }, + "device": { + "ignore_above": 1024, + "type": "keyword" + }, + "directory": { + "ignore_above": 1024, + "type": "keyword" + }, + "drive_letter": { + "ignore_above": 1, + "type": "keyword" + }, + "extension": { + "ignore_above": 1024, + "type": "keyword" + }, + "gid": { + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "ignore_above": 1024, + "type": "keyword" + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha512": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "inode": { + "ignore_above": 1024, + "type": "keyword" + }, + "mime_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "mode": { + "ignore_above": 1024, + "type": "keyword" + }, + "mtime": { + "type": "date" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "owner": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "pe": { + "properties": { + "architecture": { + "ignore_above": 1024, + "type": "keyword" + }, + "company": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "file_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "imphash": { + "ignore_above": 1024, + "type": "keyword" + }, + "original_file_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "product": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "size": { + "type": "long" + }, + "target_path": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "uid": { + "ignore_above": 1024, + "type": "keyword" + }, + "x509": { + "properties": { + "alternative_names": { + "ignore_above": 1024, + "type": "keyword" + }, + "issuer": { + "properties": { + "common_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country": { + "ignore_above": 1024, + "type": "keyword" + }, + "distinguished_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "locality": { + "ignore_above": 1024, + "type": "keyword" + }, + "organization": { + "ignore_above": 1024, + "type": "keyword" + }, + "organizational_unit": { + "ignore_above": 1024, + "type": "keyword" + }, + "state_or_province": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "not_after": { + "type": "date" + }, + "not_before": { + "type": "date" + }, + "public_key_algorithm": { + "ignore_above": 1024, + "type": "keyword" + }, + "public_key_curve": { + "ignore_above": 1024, + "type": "keyword" + }, + "public_key_exponent": { + "doc_values": false, + "index": false, + "type": "long" + }, + "public_key_size": { + "type": "long" + }, + "serial_number": { + "ignore_above": 1024, + "type": "keyword" + }, + "signature_algorithm": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject": { + "properties": { + "common_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country": { + "ignore_above": 1024, + "type": "keyword" + }, + "distinguished_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "locality": { + "ignore_above": 1024, + "type": "keyword" + }, + "organization": { + "ignore_above": 1024, + "type": "keyword" + }, + "organizational_unit": { + "ignore_above": 1024, + "type": "keyword" + }, + "state_or_province": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "version_number": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "host": { + "properties": { + "architecture": { + "ignore_above": 1024, + "type": "keyword" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "properties": { + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "uptime": { + "type": "long" + }, + "user": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "roles": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "http": { + "properties": { + "request": { + "properties": { + "body": { + "properties": { + "bytes": { + "type": "long" + }, + "content": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "bytes": { + "type": "long" + }, + "method": { + "ignore_above": 1024, + "type": "keyword" + }, + "mime_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "referrer": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "response": { + "properties": { + "body": { + "properties": { + "bytes": { + "type": "long" + }, + "content": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "bytes": { + "type": "long" + }, + "mime_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "status_code": { + "type": "long" + } + } + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "labels": { + "type": "object" + }, + "log": { + "properties": { + "file": { + "properties": { + "path": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "level": { + "ignore_above": 1024, + "type": "keyword" + }, + "logger": { + "ignore_above": 1024, + "type": "keyword" + }, + "origin": { + "properties": { + "file": { + "properties": { + "line": { + "type": "integer" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "function": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "original": { + "doc_values": false, + "ignore_above": 1024, + "index": false, + "type": "keyword" + }, + "syslog": { + "properties": { + "facility": { + "properties": { + "code": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "priority": { + "type": "long" + }, + "severity": { + "properties": { + "code": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + }, + "type": "object" + } + } + }, + "message": { + "norms": false, + "type": "text" + }, + "network": { + "properties": { + "application": { + "ignore_above": 1024, + "type": "keyword" + }, + "bytes": { + "type": "long" + }, + "community_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "direction": { + "ignore_above": 1024, + "type": "keyword" + }, + "forwarded_ip": { + "type": "ip" + }, + "iana_number": { + "ignore_above": 1024, + "type": "keyword" + }, + "inner": { + "properties": { + "vlan": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + }, + "type": "object" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "packets": { + "type": "long" + }, + "protocol": { + "ignore_above": 1024, + "type": "keyword" + }, + "transport": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "vlan": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "observer": { + "properties": { + "egress": { + "properties": { + "interface": { + "properties": { + "alias": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "vlan": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "zone": { + "ignore_above": 1024, + "type": "keyword" + } + }, + "type": "object" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "ingress": { + "properties": { + "interface": { + "properties": { + "alias": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "vlan": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "zone": { + "ignore_above": 1024, + "type": "keyword" + } + }, + "type": "object" + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "properties": { + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "product": { + "ignore_above": 1024, + "type": "keyword" + }, + "serial_number": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "vendor": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "organization": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "package": { + "properties": { + "architecture": { + "ignore_above": 1024, + "type": "keyword" + }, + "build_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "checksum": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "install_scope": { + "ignore_above": 1024, + "type": "keyword" + }, + "installed": { + "type": "date" + }, + "license": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + }, + "size": { + "type": "long" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "process": { + "properties": { + "args": { + "ignore_above": 1024, + "type": "keyword" + }, + "args_count": { + "type": "long" + }, + "code_signature": { + "properties": { + "exists": { + "type": "boolean" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "trusted": { + "type": "boolean" + }, + "valid": { + "type": "boolean" + } + } + }, + "command_line": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "entity_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "executable": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "exit_code": { + "type": "long" + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha512": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "parent": { + "properties": { + "args": { + "ignore_above": 1024, + "type": "keyword" + }, + "args_count": { + "type": "long" + }, + "code_signature": { + "properties": { + "exists": { + "type": "boolean" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "trusted": { + "type": "boolean" + }, + "valid": { + "type": "boolean" + } + } + }, + "command_line": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "entity_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "executable": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "exit_code": { + "type": "long" + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha512": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "pe": { + "properties": { + "architecture": { + "ignore_above": 1024, + "type": "keyword" + }, + "company": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "file_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "imphash": { + "ignore_above": 1024, + "type": "keyword" + }, + "original_file_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "product": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "pgid": { + "type": "long" + }, + "pid": { + "type": "long" + }, + "ppid": { + "type": "long" + }, + "start": { + "type": "date" + }, + "thread": { + "properties": { + "id": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "title": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "uptime": { + "type": "long" + }, + "working_directory": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "pe": { + "properties": { + "architecture": { + "ignore_above": 1024, + "type": "keyword" + }, + "company": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "file_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "imphash": { + "ignore_above": 1024, + "type": "keyword" + }, + "original_file_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "product": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "pgid": { + "type": "long" + }, + "pid": { + "type": "long" + }, + "ppid": { + "type": "long" + }, + "start": { + "type": "date" + }, + "thread": { + "properties": { + "id": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "title": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "uptime": { + "type": "long" + }, + "working_directory": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "registry": { + "properties": { + "data": { + "properties": { + "bytes": { + "ignore_above": 1024, + "type": "keyword" + }, + "strings": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hive": { + "ignore_above": 1024, + "type": "keyword" + }, + "key": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "value": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "related": { + "properties": { + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "hosts": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "ip" + }, + "user": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "rule": { + "properties": { + "author": { + "ignore_above": 1024, + "type": "keyword" + }, + "category": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "license": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + }, + "ruleset": { + "ignore_above": 1024, + "type": "keyword" + }, + "uuid": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "server": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "as": { + "properties": { + "number": { + "type": "long" + }, + "organization": { + "properties": { + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "bytes": { + "type": "long" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "nat": { + "properties": { + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + } + } + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + }, + "registered_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "subdomain": { + "ignore_above": 1024, + "type": "keyword" + }, + "top_level_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "roles": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "service": { + "properties": { + "ephemeral_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "node": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "source": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "as": { + "properties": { + "number": { + "type": "long" + }, + "organization": { + "properties": { + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "bytes": { + "type": "long" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "nat": { + "properties": { + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + } + } + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + }, + "registered_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "subdomain": { + "ignore_above": 1024, + "type": "keyword" + }, + "top_level_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "roles": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "span": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "tags": { + "ignore_above": 1024, + "type": "keyword" + }, + "threat": { + "properties": { + "framework": { + "ignore_above": 1024, + "type": "keyword" + }, + "tactic": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "technique": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + }, + "subtechnique": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + } + }, + "tls": { + "properties": { + "cipher": { + "ignore_above": 1024, + "type": "keyword" + }, + "client": { + "properties": { + "certificate": { + "ignore_above": 1024, + "type": "keyword" + }, + "certificate_chain": { + "ignore_above": 1024, + "type": "keyword" + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "issuer": { + "ignore_above": 1024, + "type": "keyword" + }, + "ja3": { + "ignore_above": 1024, + "type": "keyword" + }, + "not_after": { + "type": "date" + }, + "not_before": { + "type": "date" + }, + "server_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject": { + "ignore_above": 1024, + "type": "keyword" + }, + "supported_ciphers": { + "ignore_above": 1024, + "type": "keyword" + }, + "x509": { + "properties": { + "alternative_names": { + "ignore_above": 1024, + "type": "keyword" + }, + "issuer": { + "properties": { + "common_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country": { + "ignore_above": 1024, + "type": "keyword" + }, + "distinguished_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "locality": { + "ignore_above": 1024, + "type": "keyword" + }, + "organization": { + "ignore_above": 1024, + "type": "keyword" + }, + "organizational_unit": { + "ignore_above": 1024, + "type": "keyword" + }, + "state_or_province": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "not_after": { + "type": "date" + }, + "not_before": { + "type": "date" + }, + "public_key_algorithm": { + "ignore_above": 1024, + "type": "keyword" + }, + "public_key_curve": { + "ignore_above": 1024, + "type": "keyword" + }, + "public_key_exponent": { + "doc_values": false, + "index": false, + "type": "long" + }, + "public_key_size": { + "type": "long" + }, + "serial_number": { + "ignore_above": 1024, + "type": "keyword" + }, + "signature_algorithm": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject": { + "properties": { + "common_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country": { + "ignore_above": 1024, + "type": "keyword" + }, + "distinguished_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "locality": { + "ignore_above": 1024, + "type": "keyword" + }, + "organization": { + "ignore_above": 1024, + "type": "keyword" + }, + "organizational_unit": { + "ignore_above": 1024, + "type": "keyword" + }, + "state_or_province": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "version_number": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "curve": { + "ignore_above": 1024, + "type": "keyword" + }, + "established": { + "type": "boolean" + }, + "next_protocol": { + "ignore_above": 1024, + "type": "keyword" + }, + "resumed": { + "type": "boolean" + }, + "server": { + "properties": { + "certificate": { + "ignore_above": 1024, + "type": "keyword" + }, + "certificate_chain": { + "ignore_above": 1024, + "type": "keyword" + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "issuer": { + "ignore_above": 1024, + "type": "keyword" + }, + "ja3s": { + "ignore_above": 1024, + "type": "keyword" + }, + "not_after": { + "type": "date" + }, + "not_before": { + "type": "date" + }, + "subject": { + "ignore_above": 1024, + "type": "keyword" + }, + "x509": { + "properties": { + "alternative_names": { + "ignore_above": 1024, + "type": "keyword" + }, + "issuer": { + "properties": { + "common_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country": { + "ignore_above": 1024, + "type": "keyword" + }, + "distinguished_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "locality": { + "ignore_above": 1024, + "type": "keyword" + }, + "organization": { + "ignore_above": 1024, + "type": "keyword" + }, + "organizational_unit": { + "ignore_above": 1024, + "type": "keyword" + }, + "state_or_province": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "not_after": { + "type": "date" + }, + "not_before": { + "type": "date" + }, + "public_key_algorithm": { + "ignore_above": 1024, + "type": "keyword" + }, + "public_key_curve": { + "ignore_above": 1024, + "type": "keyword" + }, + "public_key_exponent": { + "doc_values": false, + "index": false, + "type": "long" + }, + "public_key_size": { + "type": "long" + }, + "serial_number": { + "ignore_above": 1024, + "type": "keyword" + }, + "signature_algorithm": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject": { + "properties": { + "common_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country": { + "ignore_above": 1024, + "type": "keyword" + }, + "distinguished_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "locality": { + "ignore_above": 1024, + "type": "keyword" + }, + "organization": { + "ignore_above": 1024, + "type": "keyword" + }, + "organizational_unit": { + "ignore_above": 1024, + "type": "keyword" + }, + "state_or_province": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "version_number": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + }, + "version_protocol": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "trace": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "transaction": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "url": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "extension": { + "ignore_above": 1024, + "type": "keyword" + }, + "fragment": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "password": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "port": { + "type": "long" + }, + "query": { + "ignore_above": 1024, + "type": "keyword" + }, + "registered_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "scheme": { + "ignore_above": 1024, + "type": "keyword" + }, + "subdomain": { + "ignore_above": 1024, + "type": "keyword" + }, + "top_level_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "username": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "user": { + "properties": { + "changes": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "roles": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "effective": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "roles": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "roles": { + "ignore_above": 1024, + "type": "keyword" + }, + "target": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "roles": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "user_agent": { + "properties": { + "device": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "properties": { + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "vulnerability": { + "properties": { + "category": { + "ignore_above": 1024, + "type": "keyword" + }, + "classification": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "enumeration": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + }, + "report_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "scanner": { + "properties": { + "vendor": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "score": { + "properties": { + "base": { + "type": "float" + }, + "environmental": { + "type": "float" + }, + "temporal": { + "type": "float" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "severity": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } +} \ No newline at end of file diff --git a/x-pack/plugins/rule_registry/server/index.ts b/x-pack/plugins/rule_registry/server/index.ts new file mode 100644 index 0000000000000..7c46717300819 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/index.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { schema, TypeOf } from '@kbn/config-schema'; +import { PluginInitializerContext } from 'src/core/server'; +import { RuleRegistryPlugin } from './plugin'; + +export { RuleRegistryPluginSetupContract } from './plugin'; +export { createLifecycleRuleTypeFactory } from './rule_registry/rule_type_helpers/create_lifecycle_rule_type_factory'; +export { ecsFieldMap } from './generated/ecs_field_map'; +export { pickWithPatterns } from './rule_registry/field_map/pick_with_patterns'; +export { FieldMapOf } from './types'; +export { ScopedRuleRegistryClient } from './rule_registry/create_scoped_rule_registry_client/types'; + +export const config = { + schema: schema.object({ + enabled: schema.boolean({ defaultValue: true }), + writeEnabled: schema.boolean({ defaultValue: false }), + }), +}; + +export type RuleRegistryConfig = TypeOf; + +export const plugin = (initContext: PluginInitializerContext) => + new RuleRegistryPlugin(initContext); diff --git a/x-pack/plugins/rule_registry/server/plugin.ts b/x-pack/plugins/rule_registry/server/plugin.ts new file mode 100644 index 0000000000000..9e83d938d508b --- /dev/null +++ b/x-pack/plugins/rule_registry/server/plugin.ts @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { PluginInitializerContext, Plugin, CoreSetup } from 'src/core/server'; +import { PluginSetupContract as AlertingPluginSetupContract } from '../../alerting/server'; +import { RuleRegistry } from './rule_registry'; +import { defaultIlmPolicy } from './rule_registry/defaults/ilm_policy'; +import { defaultFieldMap } from './rule_registry/defaults/field_map'; +import { RuleRegistryConfig } from '.'; + +export type RuleRegistryPluginSetupContract = RuleRegistry; + +export class RuleRegistryPlugin implements Plugin { + constructor(private readonly initContext: PluginInitializerContext) { + this.initContext = initContext; + } + + public setup( + core: CoreSetup, + plugins: { alerting: AlertingPluginSetupContract } + ): RuleRegistryPluginSetupContract { + const globalConfig = this.initContext.config.legacy.get(); + const config = this.initContext.config.get(); + + const logger = this.initContext.logger.get(); + + const rootRegistry = new RuleRegistry({ + coreSetup: core, + ilmPolicy: defaultIlmPolicy, + fieldMap: defaultFieldMap, + kibanaIndex: globalConfig.kibana.index, + name: 'alerts', + kibanaVersion: this.initContext.env.packageInfo.version, + logger: logger.get('root'), + alertingPluginSetupContract: plugins.alerting, + writeEnabled: config.writeEnabled, + }); + + return rootRegistry; + } + + public start() {} + + public stop() {} +} diff --git a/x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/index.ts b/x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/index.ts new file mode 100644 index 0000000000000..9a3d4a38d2ad6 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/index.ts @@ -0,0 +1,174 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { Either, isLeft, isRight } from 'fp-ts/lib/Either'; +import { Errors } from 'io-ts'; +import { PathReporter } from 'io-ts/lib/PathReporter'; +import { Logger, SavedObjectsClientContract } from 'kibana/server'; +import { IScopedClusterClient as ScopedClusterClient } from 'src/core/server'; +import { compact } from 'lodash'; +import { ESSearchRequest } from 'typings/elasticsearch'; +import { ClusterClientAdapter } from '../../../../event_log/server'; +import { runtimeTypeFromFieldMap, OutputOfFieldMap } from '../field_map/runtime_type_from_fieldmap'; +import { ScopedRuleRegistryClient, EventsOf } from './types'; +import { DefaultFieldMap } from '../defaults/field_map'; + +const getRuleUuids = async ({ + savedObjectsClient, + namespace, +}: { + savedObjectsClient: SavedObjectsClientContract; + namespace?: string; +}) => { + const options = { + type: 'alert', + ...(namespace ? { namespace } : {}), + }; + + const pitFinder = savedObjectsClient.createPointInTimeFinder({ + ...options, + }); + + const ruleUuids: string[] = []; + + for await (const response of pitFinder.find()) { + ruleUuids.push(...response.saved_objects.map((object) => object.id)); + } + + await pitFinder.close(); + + return ruleUuids; +}; + +const createPathReporterError = (either: Either) => { + const error = new Error(`Failed to validate alert event`); + error.stack += '\n' + PathReporter.report(either).join('\n'); + return error; +}; + +export function createScopedRuleRegistryClient({ + fieldMap, + scopedClusterClient, + savedObjectsClient, + namespace, + clusterClientAdapter, + indexAliasName, + indexTarget, + logger, + ruleData, +}: { + fieldMap: TFieldMap; + scopedClusterClient: ScopedClusterClient; + savedObjectsClient: SavedObjectsClientContract; + namespace?: string; + clusterClientAdapter: ClusterClientAdapter<{ + body: OutputOfFieldMap; + index: string; + }>; + indexAliasName: string; + indexTarget: string; + logger: Logger; + ruleData?: { + rule: { + id: string; + uuid: string; + category: string; + name: string; + }; + producer: string; + tags: string[]; + }; +}): ScopedRuleRegistryClient { + const docRt = runtimeTypeFromFieldMap(fieldMap); + + const defaults: Partial> = ruleData + ? { + 'rule.uuid': ruleData.rule.uuid, + 'rule.id': ruleData.rule.id, + 'rule.name': ruleData.rule.name, + 'rule.category': ruleData.rule.category, + 'kibana.rac.producer': ruleData.producer, + tags: ruleData.tags, + } + : {}; + + const client: ScopedRuleRegistryClient = { + search: async (searchRequest) => { + const ruleUuids = await getRuleUuids({ + savedObjectsClient, + namespace, + }); + + const response = await scopedClusterClient.asInternalUser.search({ + ...searchRequest, + index: indexTarget, + body: { + ...searchRequest.body, + query: { + bool: { + filter: [ + { terms: { 'rule.uuid': ruleUuids } }, + ...(searchRequest.body?.query ? [searchRequest.body.query] : []), + ], + }, + }, + }, + }); + + return { + body: response.body as any, + events: compact( + response.body.hits.hits.map((hit) => { + const validation = docRt.decode(hit.fields); + if (isLeft(validation)) { + const error = createPathReporterError(validation); + logger.error(error); + return undefined; + } + return docRt.encode(validation.right); + }) + ) as EventsOf, + }; + }, + index: (doc) => { + const validation = docRt.decode({ + ...doc, + ...defaults, + }); + + if (isLeft(validation)) { + throw createPathReporterError(validation); + } + + clusterClientAdapter.indexDocument({ body: validation.right, index: indexAliasName }); + }, + bulkIndex: (docs) => { + const validations = docs.map((doc) => { + return docRt.decode({ + ...doc, + ...defaults, + }); + }); + + const errors = compact( + validations.map((validation) => + isLeft(validation) ? createPathReporterError(validation) : null + ) + ); + + errors.forEach((error) => { + logger.error(error); + }); + + const operations = compact( + validations.map((validation) => (isRight(validation) ? validation.right : null)) + ).map((doc) => ({ body: doc, index: indexAliasName })); + + return clusterClientAdapter.indexDocuments(operations); + }, + }; + return client; +} diff --git a/x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/types.ts b/x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/types.ts new file mode 100644 index 0000000000000..95aa180709a51 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/types.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ESSearchRequest, ESSearchResponse } from 'typings/elasticsearch'; +import { DefaultFieldMap } from '../defaults/field_map'; +import { PatternsUnionOf, PickWithPatterns } from '../field_map/pick_with_patterns'; +import { OutputOfFieldMap } from '../field_map/runtime_type_from_fieldmap'; + +export type PrepopulatedRuleEventFields = + | 'rule.uuid' + | 'rule.id' + | 'rule.name' + | 'rule.type' + | 'rule.category' + | 'producer'; + +type FieldsOf = + | Array<{ field: PatternsUnionOf } | PatternsUnionOf> + | PatternsUnionOf; + +type Fields = Array<{ field: TPattern } | TPattern> | TPattern; + +type FieldsESSearchRequest = ESSearchRequest & { + body?: { fields: FieldsOf }; +}; + +export type EventsOf< + TFieldsESSearchRequest extends ESSearchRequest, + TFieldMap extends DefaultFieldMap +> = TFieldsESSearchRequest extends { body: { fields: infer TFields } } + ? TFields extends Fields + ? Array>> + : never + : never; + +export interface ScopedRuleRegistryClient { + search>( + request: TSearchRequest + ): Promise<{ + body: ESSearchResponse; + events: EventsOf; + }>; + index(doc: Omit, PrepopulatedRuleEventFields>): void; + bulkIndex( + doc: Array, PrepopulatedRuleEventFields>> + ): Promise; +} diff --git a/x-pack/plugins/rule_registry/server/rule_registry/defaults/field_map.ts b/x-pack/plugins/rule_registry/server/rule_registry/defaults/field_map.ts new file mode 100644 index 0000000000000..db851b7b94c76 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_registry/defaults/field_map.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ecsFieldMap } from '../../generated/ecs_field_map'; +import { pickWithPatterns } from '../field_map/pick_with_patterns'; + +export const defaultFieldMap = { + ...pickWithPatterns( + ecsFieldMap, + '@timestamp', + 'event.kind', + 'event.action', + 'rule.uuid', + 'rule.id', + 'rule.name', + 'rule.category', + 'tags' + ), + 'kibana.rac.producer': { type: 'keyword' }, + 'kibana.rac.alert.uuid': { type: 'keyword' }, + 'kibana.rac.alert.id': { type: 'keyword' }, + 'kibana.rac.alert.start': { type: 'date' }, + 'kibana.rac.alert.end': { type: 'date' }, + 'kibana.rac.alert.duration.us': { type: 'long' }, + 'kibana.rac.alert.severity.level': { type: 'keyword' }, + 'kibana.rac.alert.severity.value': { type: 'long' }, + 'kibana.rac.alert.status': { type: 'keyword' }, +} as const; + +export type DefaultFieldMap = typeof defaultFieldMap; diff --git a/x-pack/plugins/rule_registry/server/rule_registry/defaults/ilm_policy.ts b/x-pack/plugins/rule_registry/server/rule_registry/defaults/ilm_policy.ts new file mode 100644 index 0000000000000..c80f7e772f308 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_registry/defaults/ilm_policy.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ILMPolicy } from '../types'; + +export const defaultIlmPolicy: ILMPolicy = { + policy: { + phases: { + hot: { + actions: { + rollover: { + max_age: '90d', + max_size: '50gb', + }, + }, + }, + delete: { + actions: { + delete: {}, + }, + }, + }, + }, +}; diff --git a/x-pack/plugins/rule_registry/server/rule_registry/field_map/mapping_from_field_map.ts b/x-pack/plugins/rule_registry/server/rule_registry/field_map/mapping_from_field_map.ts new file mode 100644 index 0000000000000..6e4e13b01d2c5 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_registry/field_map/mapping_from_field_map.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { set } from '@elastic/safer-lodash-set'; +import { FieldMap, Mappings } from '../types'; + +export function mappingFromFieldMap(fieldMap: FieldMap): Mappings { + const mappings = { + dynamic: 'strict' as const, + properties: {}, + }; + + const fields = Object.keys(fieldMap).map((key) => { + const field = fieldMap[key]; + return { + name: key, + ...field, + }; + }); + + fields.forEach((field) => { + const { name, required, array, ...rest } = field; + + set(mappings.properties, field.name.split('.').join('.properties.'), rest); + }); + + return mappings; +} diff --git a/x-pack/plugins/rule_registry/server/rule_registry/field_map/merge_field_maps.ts b/x-pack/plugins/rule_registry/server/rule_registry/field_map/merge_field_maps.ts new file mode 100644 index 0000000000000..e15b228b0f287 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_registry/field_map/merge_field_maps.ts @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import util from 'util'; +import { FieldMap } from '../types'; + +export function mergeFieldMaps( + first: T1, + second: T2 +): T1 & T2 { + const conflicts: Array> = []; + + Object.keys(second).forEach((name) => { + const field = second[name]; + + const parts = name.split('.'); + + const parents = parts.slice(0, parts.length - 2).map((part, index, array) => { + return [...array.slice(0, index - 1), part].join('.'); + }); + + parents + .filter((parent) => first[parent] !== undefined) + .forEach((parent) => { + conflicts.push({ + [parent]: [{ type: 'object' }, first[parent]!], + }); + }); + + if (first[name]) { + conflicts.push({ + [name]: [field, first[name]], + }); + } + }); + + if (conflicts.length) { + const err = new Error(`Could not merge mapping due to conflicts`); + Object.assign(err, { conflicts: util.inspect(conflicts, { depth: null }) }); + throw err; + } + + return { + ...first, + ...second, + }; +} diff --git a/x-pack/plugins/rule_registry/server/rule_registry/field_map/pick_with_patterns.test.ts b/x-pack/plugins/rule_registry/server/rule_registry/field_map/pick_with_patterns.test.ts new file mode 100644 index 0000000000000..48ba7c873db25 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_registry/field_map/pick_with_patterns.test.ts @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { pickWithPatterns } from './pick_with_patterns'; + +describe('pickWithPatterns', () => { + const fieldMap = { + 'event.category': { type: 'keyword' }, + 'event.kind': { type: 'keyword' }, + 'destination.bytes': { + type: 'long', + array: false, + required: false, + }, + 'destination.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.geo.city_name': { + type: 'keyword', + array: false, + required: false, + }, + } as const; + + it('picks a single field', () => { + expect(Object.keys(pickWithPatterns(fieldMap, 'event.category'))).toEqual(['event.category']); + }); + + it('picks event fields', () => { + expect(Object.keys(pickWithPatterns(fieldMap, 'event.*')).sort()).toEqual([ + 'event.category', + 'event.kind', + ]); + }); + + it('picks destination.geo fields', () => { + expect(Object.keys(pickWithPatterns(fieldMap, 'destination.geo.*')).sort()).toEqual([ + 'destination.geo.city_name', + ]); + }); + + it('picks all destination fields', () => { + expect(Object.keys(pickWithPatterns(fieldMap, 'destination.*')).sort()).toEqual([ + 'destination.bytes', + 'destination.domain', + 'destination.geo.city_name', + ]); + }); + + it('picks fields from multiple patterns', () => { + expect( + Object.keys(pickWithPatterns(fieldMap, 'destination.geo.*', 'event.category')).sort() + ).toEqual(['destination.geo.city_name', 'event.category']); + }); + + it('picks all fields', () => { + expect(Object.keys(pickWithPatterns(fieldMap, '*')).sort()).toEqual([ + 'destination.bytes', + 'destination.domain', + 'destination.geo.city_name', + 'event.category', + 'event.kind', + ]); + }); +}); diff --git a/x-pack/plugins/rule_registry/server/rule_registry/field_map/pick_with_patterns.ts b/x-pack/plugins/rule_registry/server/rule_registry/field_map/pick_with_patterns.ts new file mode 100644 index 0000000000000..f8a88957fceb5 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_registry/field_map/pick_with_patterns.ts @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ValuesType, SetIntersection, OmitByValueExact } from 'utility-types'; +import { pick } from 'lodash'; + +type SplitByDot< + TPath extends string, + TPrefix extends string = '' +> = TPath extends `${infer TKey}.${infer TRest}` + ? [`${TPrefix}${TKey}.*`, ...SplitByDot] + : [`${TPrefix}${TPath}`]; + +type PatternMapOf> = { + [TKey in keyof T]: ValuesType] : never>; +}; + +export type PickWithPatterns< + T extends Record, + TPatterns extends string[] +> = OmitByValueExact< + { + [TFieldName in keyof T]: SetIntersection< + ValuesType, + PatternMapOf[TFieldName] + > extends never + ? never + : T[TFieldName]; + }, + never +>; + +export type PatternsUnionOf> = '*' | ValuesType>; + +export function pickWithPatterns< + T extends Record, + TPatterns extends Array> +>(map: T, ...patterns: TPatterns): PickWithPatterns { + const allFields = Object.keys(map); + const matchedFields = allFields.filter((field) => + patterns.some((pattern) => { + if (pattern === field) { + return true; + } + + const fieldParts = field.split('.'); + const patternParts = pattern.split('.'); + + if (patternParts.indexOf('*') !== patternParts.length - 1) { + return false; + } + + return fieldParts.every((fieldPart, index) => { + const patternPart = patternParts.length - 1 < index ? '*' : patternParts[index]; + + return fieldPart === patternPart || patternPart === '*'; + }); + }) + ); + + return (pick(map, matchedFields) as unknown) as PickWithPatterns; +} diff --git a/x-pack/plugins/rule_registry/server/rule_registry/field_map/runtime_type_from_fieldmap.test.ts b/x-pack/plugins/rule_registry/server/rule_registry/field_map/runtime_type_from_fieldmap.test.ts new file mode 100644 index 0000000000000..0acf80bfb42e5 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_registry/field_map/runtime_type_from_fieldmap.test.ts @@ -0,0 +1,95 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { runtimeTypeFromFieldMap } from './runtime_type_from_fieldmap'; + +describe('runtimeTypeFromFieldMap', () => { + const fieldmapRt = runtimeTypeFromFieldMap({ + keywordField: { type: 'keyword' }, + longField: { type: 'long' }, + requiredKeywordField: { type: 'keyword', required: true }, + multiKeywordField: { type: 'keyword', array: true }, + } as const); + + it('accepts both singular and array fields', () => { + expect( + fieldmapRt.is({ + requiredKeywordField: 'keyword', + }) + ).toBe(true); + + expect( + fieldmapRt.is({ + requiredKeywordField: ['keyword'], + }) + ).toBe(true); + + expect( + fieldmapRt.is({ + requiredKeywordField: ['keyword'], + multiKeywordField: 'keyword', + }) + ).toBe(true); + + expect( + fieldmapRt.is({ + requiredKeywordField: ['keyword'], + multiKeywordField: ['keyword'], + }) + ).toBe(true); + }); + + it('fails on invalid data types', () => { + expect( + fieldmapRt.is({ + requiredKeywordField: 2, + }) + ).toBe(false); + + expect( + fieldmapRt.is({ + requiredKeywordField: [2], + }) + ).toBe(false); + + expect( + fieldmapRt.is({ + requiredKeywordField: ['keyword'], + longField: ['keyword'], + }) + ).toBe(false); + + expect( + fieldmapRt.is({ + requiredKeywordField: ['keyword'], + longField: [3], + }) + ).toBe(true); + + expect( + fieldmapRt.is({ + requiredKeywordField: ['keyword'], + longField: 3, + }) + ).toBe(true); + }); + + it('outputs to single or array values', () => { + expect( + fieldmapRt.encode({ + requiredKeywordField: ['required'], + keywordField: 'keyword', + longField: [3, 2], + multiKeywordField: ['keyword', 'foo'], + }) + ).toEqual({ + requiredKeywordField: 'required', + keywordField: 'keyword', + longField: 3, + multiKeywordField: ['keyword', 'foo'], + }); + }); +}); diff --git a/x-pack/plugins/rule_registry/server/rule_registry/field_map/runtime_type_from_fieldmap.ts b/x-pack/plugins/rule_registry/server/rule_registry/field_map/runtime_type_from_fieldmap.ts new file mode 100644 index 0000000000000..6dc557c016d1a --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_registry/field_map/runtime_type_from_fieldmap.ts @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { mapValues, pickBy } from 'lodash'; +import * as t from 'io-ts'; +import { Mutable, PickByValueExact } from 'utility-types'; +import { FieldMap } from '../types'; + +const esFieldTypeMap = { + keyword: t.string, + text: t.string, + date: t.string, + boolean: t.boolean, + byte: t.number, + long: t.number, + integer: t.number, + short: t.number, + double: t.number, + float: t.number, + scaled_float: t.number, + unsigned_long: t.number, + flattened: t.record(t.string, t.array(t.string)), +}; + +type EsFieldTypeMap = typeof esFieldTypeMap; + +type EsFieldTypeOf = T extends keyof EsFieldTypeMap + ? EsFieldTypeMap[T] + : t.UnknownC; + +type RequiredKeysOf> = keyof PickByValueExact< + { + [key in keyof T]: T[key]['required']; + }, + true +>; + +type IntersectionTypeOf< + T extends Record +> = t.IntersectionC< + [ + t.TypeC>>, + t.PartialC<{ [key in keyof T]: T[key]['type'] }> + ] +>; + +type CastArray> = t.Type< + t.TypeOf | Array>, + Array>, + unknown +>; +type CastSingle> = t.Type< + t.TypeOf | Array>, + t.TypeOf, + unknown +>; + +const createCastArrayRt = >(type: T): CastArray => { + const union = t.union([type, t.array(type)]); + + return new t.Type('castArray', union.is, union.validate, (a) => (Array.isArray(a) ? a : [a])); +}; + +const createCastSingleRt = >(type: T): CastSingle => { + const union = t.union([type, t.array(type)]); + + return new t.Type('castSingle', union.is, union.validate, (a) => (Array.isArray(a) ? a[0] : a)); +}; + +type MapTypeValues = { + [key in keyof T]: { + required: T[key]['required']; + type: T[key]['array'] extends true + ? CastArray> + : CastSingle>; + }; +}; + +type FieldMapType = IntersectionTypeOf>; + +export type TypeOfFieldMap = Mutable>>; +export type OutputOfFieldMap = Mutable>>; + +export function runtimeTypeFromFieldMap( + fieldMap: TFieldMap +): FieldMapType { + function mapToType(fields: FieldMap) { + return mapValues(fields, (field, key) => { + const type = + field.type in esFieldTypeMap + ? esFieldTypeMap[field.type as keyof EsFieldTypeMap] + : t.unknown; + + return field.array ? createCastArrayRt(type) : createCastSingleRt(type); + }); + } + + const required = pickBy(fieldMap, (field) => field.required); + + return (t.intersection([ + t.exact(t.partial(mapToType(fieldMap))), + t.type(mapToType(required)), + ]) as unknown) as FieldMapType; +} diff --git a/x-pack/plugins/rule_registry/server/rule_registry/index.ts b/x-pack/plugins/rule_registry/server/rule_registry/index.ts new file mode 100644 index 0000000000000..f1d24550ade0a --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_registry/index.ts @@ -0,0 +1,240 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { CoreSetup, Logger, RequestHandlerContext } from 'kibana/server'; +import { inspect } from 'util'; +import { SpacesServiceStart } from '../../../spaces/server'; +import { + ActionVariable, + AlertInstanceState, + AlertTypeParams, + AlertTypeState, +} from '../../../alerting/common'; +import { createReadySignal, ClusterClientAdapter } from '../../../event_log/server'; +import { FieldMap, ILMPolicy } from './types'; +import { RuleParams, RuleType } from '../types'; +import { mergeFieldMaps } from './field_map/merge_field_maps'; +import { OutputOfFieldMap } from './field_map/runtime_type_from_fieldmap'; +import { mappingFromFieldMap } from './field_map/mapping_from_field_map'; +import { PluginSetupContract as AlertingPluginSetupContract } from '../../../alerting/server'; +import { createScopedRuleRegistryClient } from './create_scoped_rule_registry_client'; +import { DefaultFieldMap } from './defaults/field_map'; +import { ScopedRuleRegistryClient } from './create_scoped_rule_registry_client/types'; + +interface RuleRegistryOptions { + kibanaIndex: string; + kibanaVersion: string; + name: string; + logger: Logger; + coreSetup: CoreSetup; + spacesStart?: SpacesServiceStart; + fieldMap: TFieldMap; + ilmPolicy: ILMPolicy; + alertingPluginSetupContract: AlertingPluginSetupContract; + writeEnabled: boolean; +} + +export class RuleRegistry { + private readonly esAdapter: ClusterClientAdapter<{ + body: OutputOfFieldMap; + index: string; + }>; + private readonly children: Array> = []; + + constructor(private readonly options: RuleRegistryOptions) { + const { logger, coreSetup } = options; + + const { wait, signal } = createReadySignal(); + + this.esAdapter = new ClusterClientAdapter<{ + body: OutputOfFieldMap; + index: string; + }>({ + wait, + elasticsearchClientPromise: coreSetup + .getStartServices() + .then(([{ elasticsearch }]) => elasticsearch.client.asInternalUser), + logger: logger.get('esAdapter'), + }); + + if (this.options.writeEnabled) { + this.initialize() + .then(() => { + this.options.logger.debug('Bootstrapped alerts index'); + signal(true); + }) + .catch((err) => { + logger.error(inspect(err, { depth: null })); + signal(false); + }); + } else { + logger.debug('Write disabled, indices are not being bootstrapped'); + } + } + + private getEsNames() { + const base = [this.options.kibanaIndex, this.options.name]; + const indexTarget = `${base.join('-')}*`; + const indexAliasName = [...base, this.options.kibanaVersion.toLowerCase()].join('-'); + const policyName = [...base, 'policy'].join('-'); + + return { + indexAliasName, + indexTarget, + policyName, + }; + } + + private async initialize() { + const { indexAliasName, policyName } = this.getEsNames(); + + const ilmPolicyExists = await this.esAdapter.doesIlmPolicyExist(policyName); + + if (!ilmPolicyExists) { + await this.esAdapter.createIlmPolicy( + policyName, + (this.options.ilmPolicy as unknown) as Record + ); + } + + const templateExists = await this.esAdapter.doesIndexTemplateExist(indexAliasName); + + if (!templateExists) { + await this.esAdapter.createIndexTemplate(indexAliasName, { + index_patterns: [`${indexAliasName}-*`], + settings: { + number_of_shards: 1, + auto_expand_replicas: '0-1', + 'index.lifecycle.name': policyName, + 'index.lifecycle.rollover_alias': indexAliasName, + 'sort.field': '@timestamp', + 'sort.order': 'desc', + }, + mappings: mappingFromFieldMap(this.options.fieldMap), + }); + } + + const aliasExists = await this.esAdapter.doesAliasExist(indexAliasName); + + if (!aliasExists) { + await this.esAdapter.createIndex(`${indexAliasName}-000001`, { + aliases: { + [indexAliasName]: { + is_write_index: true, + }, + }, + }); + } + } + + createScopedRuleRegistryClient({ + context, + }: { + context: RequestHandlerContext; + }): ScopedRuleRegistryClient | undefined { + if (!this.options.writeEnabled) { + return undefined; + } + const { indexAliasName, indexTarget } = this.getEsNames(); + + return createScopedRuleRegistryClient({ + savedObjectsClient: context.core.savedObjects.getClient({ includedHiddenTypes: ['alert'] }), + scopedClusterClient: context.core.elasticsearch.client, + clusterClientAdapter: this.esAdapter, + fieldMap: this.options.fieldMap, + indexAliasName, + indexTarget, + logger: this.options.logger, + }); + } + + registerType( + type: RuleType + ) { + const logger = this.options.logger.get(type.id); + + const { indexAliasName, indexTarget } = this.getEsNames(); + + this.options.alertingPluginSetupContract.registerType< + AlertTypeParams, + AlertTypeState, + AlertInstanceState, + { [key in TActionVariable['name']]: any }, + string + >({ + ...type, + executor: async (executorOptions) => { + const { services, namespace, alertId, name, tags } = executorOptions; + + const rule = { + id: type.id, + uuid: alertId, + category: type.name, + name, + }; + + const producer = type.producer; + + return type.executor({ + ...executorOptions, + rule, + producer, + services: { + ...services, + logger, + ...(this.options.writeEnabled + ? { + scopedRuleRegistryClient: createScopedRuleRegistryClient({ + savedObjectsClient: services.savedObjectsClient, + scopedClusterClient: services.scopedClusterClient, + clusterClientAdapter: this.esAdapter, + fieldMap: this.options.fieldMap, + indexAliasName, + indexTarget, + namespace, + ruleData: { + producer, + rule, + tags, + }, + logger: this.options.logger, + }), + } + : {}), + }, + }); + }, + }); + } + + create({ + name, + fieldMap, + ilmPolicy, + }: { + name: string; + fieldMap: TNextFieldMap; + ilmPolicy?: ILMPolicy; + }): RuleRegistry { + const mergedFieldMap = fieldMap + ? mergeFieldMaps(this.options.fieldMap, fieldMap) + : this.options.fieldMap; + + const child = new RuleRegistry({ + ...this.options, + logger: this.options.logger.get(name), + name: [this.options.name, name].filter(Boolean).join('-'), + fieldMap: mergedFieldMap, + ...(ilmPolicy ? { ilmPolicy } : {}), + }); + + this.children.push(child); + + // @ts-expect-error could be instantiated with a different subtype of constraint + return child; + } +} diff --git a/x-pack/plugins/rule_registry/server/rule_registry/rule_type_helpers/create_lifecycle_rule_type_factory.ts b/x-pack/plugins/rule_registry/server/rule_registry/rule_type_helpers/create_lifecycle_rule_type_factory.ts new file mode 100644 index 0000000000000..9c64e85f839bb --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_registry/rule_type_helpers/create_lifecycle_rule_type_factory.ts @@ -0,0 +1,235 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import * as t from 'io-ts'; +import { isLeft } from 'fp-ts/lib/Either'; +import v4 from 'uuid/v4'; +import { AlertInstance } from '../../../../alerting/server'; +import { ActionVariable, AlertInstanceState } from '../../../../alerting/common'; +import { RuleParams, RuleType } from '../../types'; +import { DefaultFieldMap } from '../defaults/field_map'; +import { OutputOfFieldMap } from '../field_map/runtime_type_from_fieldmap'; +import { PrepopulatedRuleEventFields } from '../create_scoped_rule_registry_client/types'; +import { RuleRegistry } from '..'; + +type UserDefinedAlertFields = Omit< + OutputOfFieldMap, + PrepopulatedRuleEventFields | 'kibana.rac.alert.id' | 'kibana.rac.alert.uuid' | '@timestamp' +>; + +type LifecycleAlertService< + TFieldMap extends DefaultFieldMap, + TActionVariable extends ActionVariable +> = (alert: { + id: string; + fields: UserDefinedAlertFields; +}) => AlertInstance; + +type CreateLifecycleRuleType = < + TRuleParams extends RuleParams, + TActionVariable extends ActionVariable +>( + type: RuleType< + TFieldMap, + TRuleParams, + TActionVariable, + { alertWithLifecycle: LifecycleAlertService } + > +) => RuleType; + +const trackedAlertStateRt = t.type({ + alertId: t.string, + alertUuid: t.string, + started: t.string, +}); + +const wrappedStateRt = t.type({ + wrapped: t.record(t.string, t.unknown), + trackedAlerts: t.record(t.string, trackedAlertStateRt), +}); + +export function createLifecycleRuleTypeFactory< + TRuleRegistry extends RuleRegistry +>(): TRuleRegistry extends RuleRegistry + ? CreateLifecycleRuleType + : never; + +export function createLifecycleRuleTypeFactory(): CreateLifecycleRuleType { + return (type) => { + return { + ...type, + executor: async (options) => { + const { + services: { scopedRuleRegistryClient, alertInstanceFactory, logger }, + state: previousState, + rule, + } = options; + + const decodedState = wrappedStateRt.decode(previousState); + + const state = isLeft(decodedState) + ? { + wrapped: previousState, + trackedAlerts: {}, + } + : decodedState.right; + + const currentAlerts: Record< + string, + UserDefinedAlertFields & { 'kibana.rac.alert.id': string } + > = {}; + + const timestamp = options.startedAt.toISOString(); + + const nextWrappedState = await type.executor({ + ...options, + state: state.wrapped, + services: { + ...options.services, + alertWithLifecycle: ({ id, fields }) => { + currentAlerts[id] = { + ...fields, + 'kibana.rac.alert.id': id, + }; + return alertInstanceFactory(id); + }, + }, + }); + + const currentAlertIds = Object.keys(currentAlerts); + const trackedAlertIds = Object.keys(state.trackedAlerts); + const newAlertIds = currentAlertIds.filter((alertId) => !trackedAlertIds.includes(alertId)); + + const allAlertIds = [...new Set(currentAlertIds.concat(trackedAlertIds))]; + + const trackedAlertStatesOfRecovered = Object.values(state.trackedAlerts).filter( + (trackedAlertState) => !currentAlerts[trackedAlertState.alertId] + ); + + logger.debug( + `Tracking ${allAlertIds.length} alerts (${newAlertIds.length} new, ${trackedAlertStatesOfRecovered.length} recovered)` + ); + + const alertsDataMap: Record> = { + ...currentAlerts, + }; + + if (scopedRuleRegistryClient && trackedAlertStatesOfRecovered.length) { + const { events } = await scopedRuleRegistryClient.search({ + body: { + query: { + bool: { + filter: [ + { + term: { + 'rule.uuid': rule.uuid, + }, + }, + { + terms: { + 'kibana.rac.alert.uuid': trackedAlertStatesOfRecovered.map( + (trackedAlertState) => trackedAlertState.alertUuid + ), + }, + }, + ], + }, + }, + size: trackedAlertStatesOfRecovered.length, + collapse: { + field: 'kibana.rac.alert.uuid', + }, + _source: false, + fields: ['*'], + sort: { + '@timestamp': 'desc' as const, + }, + }, + }); + + events.forEach((event) => { + const alertId = event['kibana.rac.alert.id']!; + alertsDataMap[alertId] = event; + }); + } + + const eventsToIndex: Array> = allAlertIds.map( + (alertId) => { + const alertData = alertsDataMap[alertId]; + + if (!alertData) { + logger.warn(`Could not find alert data for ${alertId}`); + } + + const event: OutputOfFieldMap = { + ...alertData, + '@timestamp': timestamp, + 'event.kind': 'state', + 'kibana.rac.alert.id': alertId, + }; + + const isNew = !state.trackedAlerts[alertId]; + const isRecovered = !currentAlerts[alertId]; + const isActiveButNotNew = !isNew && !isRecovered; + const isActive = !isRecovered; + + const { alertUuid, started } = state.trackedAlerts[alertId] ?? { + alertUuid: v4(), + started: timestamp, + }; + + event['kibana.rac.alert.start'] = started; + event['kibana.rac.alert.uuid'] = alertUuid; + + if (isNew) { + event['event.action'] = 'open'; + } + + if (isRecovered) { + event['kibana.rac.alert.end'] = timestamp; + event['event.action'] = 'close'; + event['kibana.rac.alert.status'] = 'closed'; + } + + if (isActiveButNotNew) { + event['event.action'] = 'active'; + } + + if (isActive) { + event['kibana.rac.alert.status'] = 'open'; + } + + event['kibana.rac.alert.duration.us'] = + (options.startedAt.getTime() - new Date(event['kibana.rac.alert.start']!).getTime()) * + 1000; + + return event; + } + ); + + if (eventsToIndex.length && scopedRuleRegistryClient) { + await scopedRuleRegistryClient.bulkIndex(eventsToIndex); + } + + const nextTrackedAlerts = Object.fromEntries( + eventsToIndex + .filter((event) => event['kibana.rac.alert.status'] !== 'closed') + .map((event) => { + const alertId = event['kibana.rac.alert.id']!; + const alertUuid = event['kibana.rac.alert.uuid']!; + const started = new Date(event['kibana.rac.alert.start']!).toISOString(); + return [alertId, { alertId, alertUuid, started }]; + }) + ); + + return { + wrapped: nextWrappedState, + trackedAlerts: nextTrackedAlerts, + }; + }, + }; + }; +} diff --git a/x-pack/plugins/rule_registry/server/rule_registry/types.ts b/x-pack/plugins/rule_registry/server/rule_registry/types.ts new file mode 100644 index 0000000000000..f6baf8bcecbd0 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_registry/types.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export interface Mappings { + dynamic: 'strict' | boolean; + properties: Record; +} + +enum ILMPolicyPhase { + hot = 'hot', + delete = 'delete', +} + +enum ILMPolicyAction { + rollover = 'rollover', + delete = 'delete', +} + +interface ILMActionOptions { + [ILMPolicyAction.rollover]: { + max_size: string; + max_age: string; + }; + [ILMPolicyAction.delete]: {}; +} + +export interface ILMPolicy { + policy: { + phases: Record< + ILMPolicyPhase, + { + actions: { + [key in keyof ILMActionOptions]?: ILMActionOptions[key]; + }; + } + >; + }; +} + +export type FieldMap = Record; diff --git a/x-pack/plugins/rule_registry/server/types.ts b/x-pack/plugins/rule_registry/server/types.ts new file mode 100644 index 0000000000000..e6b53a8558964 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/types.ts @@ -0,0 +1,100 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { Type, TypeOf } from '@kbn/config-schema'; +import { Logger } from 'kibana/server'; +import { + ActionVariable, + AlertInstanceContext, + AlertInstanceState, + AlertTypeParams, + AlertTypeState, +} from '../../alerting/common'; +import { ActionGroup, AlertExecutorOptions } from '../../alerting/server'; +import { RuleRegistry } from './rule_registry'; +import { ScopedRuleRegistryClient } from './rule_registry/create_scoped_rule_registry_client/types'; +import { DefaultFieldMap } from './rule_registry/defaults/field_map'; + +export type RuleParams = Type; + +type TypeOfRuleParams = TypeOf; + +type RuleExecutorServices< + TFieldMap extends DefaultFieldMap, + TActionVariable extends ActionVariable +> = AlertExecutorOptions< + AlertTypeParams, + AlertTypeState, + AlertInstanceState, + { [key in TActionVariable['name']]: any }, + string +>['services'] & { + logger: Logger; + scopedRuleRegistryClient?: ScopedRuleRegistryClient; +}; + +type PassthroughAlertExecutorOptions = Pick< + AlertExecutorOptions< + AlertTypeParams, + AlertTypeState, + AlertInstanceState, + AlertInstanceContext, + string + >, + 'previousStartedAt' | 'startedAt' | 'state' +>; + +type RuleExecutorFunction< + TFieldMap extends DefaultFieldMap, + TRuleParams extends RuleParams, + TActionVariable extends ActionVariable, + TAdditionalRuleExecutorServices extends Record +> = ( + options: PassthroughAlertExecutorOptions & { + services: RuleExecutorServices & TAdditionalRuleExecutorServices; + params: TypeOfRuleParams; + rule: { + id: string; + uuid: string; + name: string; + category: string; + }; + producer: string; + } +) => Promise>; + +interface RuleTypeBase { + id: string; + name: string; + actionGroups: Array>; + defaultActionGroupId: string; + producer: string; + minimumLicenseRequired: 'basic' | 'gold' | 'trial'; +} + +export type RuleType< + TFieldMap extends DefaultFieldMap, + TRuleParams extends RuleParams, + TActionVariable extends ActionVariable, + TAdditionalRuleExecutorServices extends Record = {} +> = RuleTypeBase & { + validate: { + params: TRuleParams; + }; + actionVariables: { + context: TActionVariable[]; + }; + executor: RuleExecutorFunction< + TFieldMap, + TRuleParams, + TActionVariable, + TAdditionalRuleExecutorServices + >; +}; + +export type FieldMapOf< + TRuleRegistry extends RuleRegistry +> = TRuleRegistry extends RuleRegistry ? TFieldMap : never; diff --git a/x-pack/plugins/rule_registry/tsconfig.json b/x-pack/plugins/rule_registry/tsconfig.json new file mode 100644 index 0000000000000..2961abe6cfecd --- /dev/null +++ b/x-pack/plugins/rule_registry/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "composite": true, + "outDir": "./target/types", + "emitDeclarationOnly": true, + "declaration": true, + "declarationMap": true + }, + "include": ["common/**/*", "server/**/*", "../../../typings/**/*"], + "references": [ + { "path": "../../../src/core/tsconfig.json" }, + { "path": "../alerting/tsconfig.json" }, + ] +} diff --git a/x-pack/scripts/functional_tests.js b/x-pack/scripts/functional_tests.js index 98b74a1566ce3..af441171ec310 100644 --- a/x-pack/scripts/functional_tests.js +++ b/x-pack/scripts/functional_tests.js @@ -30,6 +30,7 @@ require('@kbn/test').runTestsCli([ require.resolve('../test/case_api_integration/basic/config.ts'), require.resolve('../test/apm_api_integration/basic/config.ts'), require.resolve('../test/apm_api_integration/trial/config.ts'), + require.resolve('../test/apm_api_integration/rules/config.ts'), require.resolve('../test/detection_engine_api_integration/security_and_spaces/config.ts'), require.resolve('../test/detection_engine_api_integration/basic/config.ts'), require.resolve('../test/lists_api_integration/security_and_spaces/config.ts'), diff --git a/x-pack/test/apm_api_integration/common/config.ts b/x-pack/test/apm_api_integration/common/config.ts index 04ce83323ee66..732f14d2a7bc8 100644 --- a/x-pack/test/apm_api_integration/common/config.ts +++ b/x-pack/test/apm_api_integration/common/config.ts @@ -18,6 +18,7 @@ import { registry } from './registry'; interface Config { name: APMFtrConfigName; license: 'basic' | 'trial'; + kibanaConfig?: Record; } const supertestAsApmUser = (kibanaServer: UrlObject, apmUser: ApmUser) => async ( @@ -37,7 +38,7 @@ const supertestAsApmUser = (kibanaServer: UrlObject, apmUser: ApmUser) => async }; export function createTestConfig(config: Config) { - const { license, name } = config; + const { license, name, kibanaConfig } = config; return async ({ readConfigFile }: FtrConfigProviderContext) => { const xPackAPITestsConfig = await readConfigFile( @@ -79,7 +80,15 @@ export function createTestConfig(config: Config) { ...xPackAPITestsConfig.get('esTestCluster'), license, }, - kbnTestServer: xPackAPITestsConfig.get('kbnTestServer'), + kbnTestServer: { + ...xPackAPITestsConfig.get('kbnTestServer'), + serverArgs: [ + ...xPackAPITestsConfig.get('kbnTestServer.serverArgs'), + ...(kibanaConfig + ? Object.entries(kibanaConfig).map(([key, value]) => `--${key}=${value}`) + : []), + ], + }, }; }; } diff --git a/x-pack/test/apm_api_integration/configs/index.ts b/x-pack/test/apm_api_integration/configs/index.ts index 4eeb57e3c86c4..91437a2d22e27 100644 --- a/x-pack/test/apm_api_integration/configs/index.ts +++ b/x-pack/test/apm_api_integration/configs/index.ts @@ -15,6 +15,12 @@ const apmFtrConfigs = { trial: { license: 'trial' as const, }, + rules: { + license: 'trial' as const, + kibanaConfig: { + 'xpack.ruleRegistry.writeEnabled': 'true', + }, + }, }; export type APMFtrConfigName = keyof typeof apmFtrConfigs; diff --git a/x-pack/test/apm_api_integration/rules/config.ts b/x-pack/test/apm_api_integration/rules/config.ts new file mode 100644 index 0000000000000..9830d516eb80a --- /dev/null +++ b/x-pack/test/apm_api_integration/rules/config.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { configs } from '../configs'; + +export default configs.rules; diff --git a/x-pack/test/apm_api_integration/tests/alerts/rule_registry.ts b/x-pack/test/apm_api_integration/tests/alerts/rule_registry.ts new file mode 100644 index 0000000000000..97026d126d2a1 --- /dev/null +++ b/x-pack/test/apm_api_integration/tests/alerts/rule_registry.ts @@ -0,0 +1,387 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { get, merge, omit } from 'lodash'; +import { FtrProviderContext } from '../../common/ftr_provider_context'; +import { registry } from '../../common/registry'; + +interface Alert { + schedule: { + interval: string; + }; + updatedAt: string; + executionStatus: { + lastExecutionDate: string; + status: string; + }; + updatedBy: string; + id: string; + params: Record; + scheduledTaskId: string; +} + +export default function ApiTest({ getService }: FtrProviderContext) { + const supertest = getService('supertestAsApmWriteUser'); + const es = getService('es'); + + const MAX_POLLS = 5; + const BULK_INDEX_DELAY = 1000; + const INDEXING_DELAY = 5000; + + const ALERTS_INDEX_TARGET = '.kibana-alerts-*-apm*'; + const APM_TRANSACTION_INDEX_NAME = 'apm-8.0.0-transaction'; + + const createTransactionEvent = (override: Record) => { + const now = Date.now(); + + const time = now - INDEXING_DELAY; + + return merge( + { + '@timestamp': new Date(time).toISOString(), + service: { + name: 'opbeans-go', + }, + event: { + outcome: 'success', + }, + transaction: { + duration: { + us: 1000000, + }, + type: 'request', + }, + processor: { + event: 'transaction', + }, + observer: { + version_major: 7, + }, + }, + override + ); + }; + + async function waitUntilNextExecution( + alert: Alert, + intervalInSeconds: number = 1, + count: number = 0 + ): Promise { + await new Promise((resolve) => { + setTimeout(resolve, intervalInSeconds * 1000); + }); + + const { body, status } = await supertest + .get(`/api/alerts/alert/${alert.id}`) + .set('kbn-xsrf', 'foo'); + + if (status >= 300) { + const error = new Error('Error getting alert'); + Object.assign(error, { response: { body, status } }); + throw error; + } + + const nextAlert = body as Alert; + + if (nextAlert.executionStatus.lastExecutionDate !== alert.executionStatus.lastExecutionDate) { + await new Promise((resolve) => { + setTimeout(resolve, BULK_INDEX_DELAY); + }); + await es.indices.refresh({ + index: ALERTS_INDEX_TARGET, + }); + + return nextAlert; + } + + if (count >= MAX_POLLS) { + throw new Error('Maximum number of polls exceeded'); + } + + return waitUntilNextExecution(alert, intervalInSeconds, count + 1); + } + + registry.when('Rule registry with write enabled', { config: 'rules', archives: [] }, () => { + it('bootstraps the apm alert indices', async () => { + const { body } = await es.indices.get({ + index: ALERTS_INDEX_TARGET, + expand_wildcards: 'open', + allow_no_indices: false, + }); + + const indices = Object.entries(body).map(([indexName, index]) => { + return { + indexName, + index, + }; + }); + + const indexNames = indices.map((index) => index.indexName); + + const apmIndex = indices[0]; + + // make sure it only creates one index + expect(indices.length).to.be(1); + + const apmIndexName = apmIndex.indexName; + + expect(apmIndexName.split('-').includes('observability')).to.be(true); + expect(apmIndexName.split('-').includes('apm')).to.be(true); + + expect(indexNames[0].startsWith('.kibana-alerts-observability-apm')).to.be(true); + + expect(get(apmIndex, 'index.mappings.properties.service.properties.environment.type')).to.be( + 'keyword' + ); + }); + + describe('when creating a rule', () => { + let createResponse: { + alert: Alert; + status: number; + }; + + before(async () => { + await es.indices.create({ + index: APM_TRANSACTION_INDEX_NAME, + body: { + mappings: { + dynamic: 'strict', + properties: { + event: { + properties: { + outcome: { + type: 'keyword', + }, + }, + }, + processor: { + properties: { + event: { + type: 'keyword', + }, + }, + }, + observer: { + properties: { + version_major: { + type: 'byte', + }, + }, + }, + service: { + properties: { + name: { + type: 'keyword', + }, + environment: { + type: 'keyword', + }, + }, + }, + transaction: { + properties: { + type: { + type: 'keyword', + }, + duration: { + properties: { + us: { + type: 'long', + }, + }, + }, + }, + }, + '@timestamp': { + type: 'date', + }, + }, + }, + }, + }); + + const body = { + params: { + threshold: 30, + windowSize: 5, + windowUnit: 'm', + transactionType: 'request', + environment: 'ENVIRONMENT_ALL', + serviceName: 'opbeans-go', + }, + consumer: 'apm', + alertTypeId: 'apm.transaction_error_rate', + schedule: { interval: '5s' }, + actions: [], + tags: ['apm', 'service.name:opbeans-go'], + notifyWhen: 'onActionGroupChange', + name: 'Transaction error rate threshold | opbeans-go', + }; + + const { body: response, status } = await supertest + .post('/api/alerts/alert') + .send(body) + .set('kbn-xsrf', 'foo'); + + createResponse = { + alert: response, + status, + }; + }); + + after(async () => { + if (createResponse.alert) { + const { body, status } = await supertest + .delete(`/api/alerts/alert/${createResponse.alert.id}`) + .set('kbn-xsrf', 'foo'); + + if (status >= 300) { + const error = new Error('Error deleting alert'); + Object.assign(error, { response: { body, status } }); + throw error; + } + } + + await es.deleteByQuery({ + index: ALERTS_INDEX_TARGET, + body: { + query: { + match_all: {}, + }, + }, + refresh: true, + }); + + await es.indices.delete({ + index: APM_TRANSACTION_INDEX_NAME, + }); + }); + + it('writes alerts data to the alert indices', async () => { + expect(createResponse.status).to.be.below(299); + + expect(createResponse.alert).not.to.be(undefined); + + let alert = await waitUntilNextExecution(createResponse.alert); + + const beforeDataResponse = await es.search({ + index: ALERTS_INDEX_TARGET, + body: { + query: { + match_all: {}, + }, + }, + size: 1, + }); + + expect(beforeDataResponse.body.hits.hits.length).to.be(0); + + await es.index({ + index: APM_TRANSACTION_INDEX_NAME, + body: createTransactionEvent({ + event: { + outcome: 'success', + }, + }), + refresh: true, + }); + + alert = await waitUntilNextExecution(alert); + + const afterInitialDataResponse = await es.search({ + index: ALERTS_INDEX_TARGET, + body: { + query: { + match_all: {}, + }, + }, + size: 1, + }); + + expect(afterInitialDataResponse.body.hits.hits.length).to.be(0); + + await es.index({ + index: APM_TRANSACTION_INDEX_NAME, + body: createTransactionEvent({ + event: { + outcome: 'failure', + }, + }), + refresh: true, + }); + + alert = await waitUntilNextExecution(alert); + + const afterViolatingDataResponse = await es.search({ + index: ALERTS_INDEX_TARGET, + body: { + query: { + match_all: {}, + }, + }, + size: 1, + }); + + expect(afterViolatingDataResponse.body.hits.hits.length).to.be(1); + + const alertEvent = afterViolatingDataResponse.body.hits.hits[0]._source as Record< + string, + any + >; + + const toCompare = omit( + alertEvent, + '@timestamp', + 'kibana.rac.alert.start', + 'kibana.rac.alert.uuid', + 'rule.uuid' + ); + + expectSnapshot(toCompare).toMatchInline(` + Object { + "event.action": "open", + "event.kind": "state", + "kibana.rac.alert.duration.us": 0, + "kibana.rac.alert.id": "apm.transaction_error_rate_opbeans-go_request", + "kibana.rac.alert.status": "open", + "kibana.rac.producer": "apm", + "rule.category": "Transaction error rate threshold", + "rule.id": "apm.transaction_error_rate", + "rule.name": "Transaction error rate threshold | opbeans-go", + "service.name": "opbeans-go", + "tags": Array [ + "apm", + "service.name:opbeans-go", + ], + "transaction.type": "request", + } + `); + }); + }); + }); + + registry.when('Rule registry with write not enabled', { config: 'basic', archives: [] }, () => { + it('does not bootstrap the apm rule indices', async () => { + const errorOrUndefined = await es.indices + .get({ + index: ALERTS_INDEX_TARGET, + expand_wildcards: 'open', + allow_no_indices: false, + }) + .then(() => {}) + .catch((error) => { + return error.toString(); + }); + + expect(errorOrUndefined).not.to.be(undefined); + + expect(errorOrUndefined).to.be(`ResponseError: index_not_found_exception`); + }); + }); +} diff --git a/x-pack/test/apm_api_integration/tests/index.ts b/x-pack/test/apm_api_integration/tests/index.ts index 7c69d5b996cea..53ec61b8d9b61 100644 --- a/x-pack/test/apm_api_integration/tests/index.ts +++ b/x-pack/test/apm_api_integration/tests/index.ts @@ -24,6 +24,10 @@ export default function apmApiIntegrationTests(providerContext: FtrProviderConte loadTestFile(require.resolve('./alerts/chart_preview')); }); + describe('alerts/rule_registry', function () { + loadTestFile(require.resolve('./alerts/rule_registry')); + }); + describe('correlations/latency_slow_transactions', function () { loadTestFile(require.resolve('./correlations/latency_slow_transactions')); });