From ec070d34ed97aa29c66a6368e60181cb7e191465 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Thu, 8 Apr 2021 20:08:10 -0600 Subject: [PATCH 1/8] [Maps] fix Kibana does not recognize a valid geo_shape index when attempting to create a Tracking Containment alert (#96633) * [Maps] fix Kibana does not recognize a valid geo_shape index when attempting to create a Tracking Containment alert * tslint * instead of forcing refresh on getIdsAndTitles, update index pattern service to add saved object to cache when index pattern is created * simplify title check * revert unneeded changes * tslint * api doc updates * fix functional test Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- ...ins-data-public.indexpatternselectprops.md | 1 - ...na-plugin-plugins-data-public.searchbar.md | 4 +- .../index_patterns/index_patterns.ts | 3 + src/plugins/data/public/public.api.md | 1 - .../index_pattern_select.tsx | 73 +++++++++++-------- .../input_control_options.ts | 2 +- 6 files changed, 50 insertions(+), 34 deletions(-) diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatternselectprops.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatternselectprops.md index 80f4832ba5643..5cfd5e1bc9929 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatternselectprops.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatternselectprops.md @@ -12,6 +12,5 @@ export declare type IndexPatternSelectProps = Required void; - maxIndexPatterns?: number; }; ``` diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchbar.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchbar.md index 7c7f2a53aca92..193a2e5a24f3f 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchbar.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchbar.md @@ -7,7 +7,7 @@ Signature: ```typescript -SearchBar: React.ComponentClass, "query" | "placeholder" | "isLoading" | "iconType" | "indexPatterns" | "filters" | "dataTestSubj" | "isClearable" | "isInvalid" | "storageKey" | "refreshInterval" | "nonKqlMode" | "nonKqlModeHelpText" | "screenTitle" | "disableLanguageSwitcher" | "autoSubmit" | "onRefresh" | "onRefreshChange" | "showQueryInput" | "showDatePicker" | "showAutoRefreshOnly" | "dateRangeFrom" | "dateRangeTo" | "isRefreshPaused" | "customSubmitButton" | "timeHistory" | "indicateNoData" | "onFiltersUpdated" | "savedQuery" | "showSaveQuery" | "onClearSavedQuery" | "showQueryBar" | "showFilterBar" | "onQueryChange" | "onQuerySubmit" | "onSaved" | "onSavedQueryUpdated">, any> & { - WrappedComponent: React.ComponentType & ReactIntl.InjectedIntlProps>; +SearchBar: React.ComponentClass, "query" | "placeholder" | "isLoading" | "iconType" | "indexPatterns" | "filters" | "dataTestSubj" | "isClearable" | "refreshInterval" | "nonKqlMode" | "nonKqlModeHelpText" | "screenTitle" | "onRefresh" | "onRefreshChange" | "showQueryInput" | "showDatePicker" | "showAutoRefreshOnly" | "dateRangeFrom" | "dateRangeTo" | "isRefreshPaused" | "customSubmitButton" | "timeHistory" | "indicateNoData" | "onFiltersUpdated" | "savedQuery" | "showSaveQuery" | "onClearSavedQuery" | "showQueryBar" | "showFilterBar" | "onQueryChange" | "onQuerySubmit" | "onSaved" | "onSavedQueryUpdated">, any> & { + WrappedComponent: React.ComponentType & ReactIntl.InjectedIntlProps>; } ``` diff --git a/src/plugins/data/common/index_patterns/index_patterns/index_patterns.ts b/src/plugins/data/common/index_patterns/index_patterns/index_patterns.ts index 805eccd1ee31b..04d2785137719 100644 --- a/src/plugins/data/common/index_patterns/index_patterns/index_patterns.ts +++ b/src/plugins/data/common/index_patterns/index_patterns/index_patterns.ts @@ -535,6 +535,9 @@ export class IndexPatternsService { }); indexPattern.id = response.id; this.indexPatternCache.set(indexPattern.id, Promise.resolve(indexPattern)); + if (this.savedObjectsCache) { + this.savedObjectsCache.push(response as SavedObject); + } return indexPattern; } diff --git a/src/plugins/data/public/public.api.md b/src/plugins/data/public/public.api.md index 05925f097de24..c80f008636ba6 100644 --- a/src/plugins/data/public/public.api.md +++ b/src/plugins/data/public/public.api.md @@ -1560,7 +1560,6 @@ export type IndexPatternSelectProps = Required, 'isLo indexPatternId: string; fieldTypes?: string[]; onNoIndexPatterns?: () => void; - maxIndexPatterns?: number; }; // Warning: (ae-missing-release-tag) "IndexPatternSpec" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) diff --git a/src/plugins/data/public/ui/index_pattern_select/index_pattern_select.tsx b/src/plugins/data/public/ui/index_pattern_select/index_pattern_select.tsx index aa36323d11bcc..04bdb7a690268 100644 --- a/src/plugins/data/public/ui/index_pattern_select/index_pattern_select.tsx +++ b/src/plugins/data/public/ui/index_pattern_select/index_pattern_select.tsx @@ -25,7 +25,6 @@ export type IndexPatternSelectProps = Required< indexPatternId: string; fieldTypes?: string[]; onNoIndexPatterns?: () => void; - maxIndexPatterns?: number; }; export type IndexPatternSelectInternalProps = IndexPatternSelectProps & { @@ -42,10 +41,6 @@ interface IndexPatternSelectState { // Needed for React.lazy // eslint-disable-next-line import/no-default-export export default class IndexPatternSelect extends Component { - static defaultProps: { - maxIndexPatterns: 1000; - }; - private isMounted: boolean = false; state: IndexPatternSelectState; @@ -67,7 +62,7 @@ export default class IndexPatternSelect extends Component { - const { fieldTypes, onNoIndexPatterns, indexPatternService } = this.props; - const indexPatterns = await indexPatternService.find( - `${searchValue}*`, - this.props.maxIndexPatterns - ); + const isCurrentSearch = () => { + return this.isMounted && searchValue === this.state.searchValue; + }; - // We need this check to handle the case where search results come back in a different - // order than they were sent out. Only load results for the most recent search. - if (searchValue !== this.state.searchValue || !this.isMounted) { + const idsAndTitles = await this.props.indexPatternService.getIdsWithTitle(); + if (!isCurrentSearch()) { return; } - const options = indexPatterns - .filter((indexPattern) => { - return fieldTypes - ? indexPattern.fields.some((field) => { - return fieldTypes.includes(field.type); - }) - : true; - }) - .map((indexPattern) => { - return { - label: indexPattern.title, - value: indexPattern.id, - }; + const options = []; + for (let i = 0; i < idsAndTitles.length; i++) { + if (!idsAndTitles[i].title.toLowerCase().includes(searchValue.toLowerCase())) { + // index pattern excluded due to title not matching search + continue; + } + + if (this.props.fieldTypes) { + try { + const indexPattern = await this.props.indexPatternService.get(idsAndTitles[i].id); + if (!isCurrentSearch()) { + return; + } + const hasRequiredFieldTypes = indexPattern.fields.some((field) => { + return this.props.fieldTypes!.includes(field.type); + }); + if (!hasRequiredFieldTypes) { + continue; + } + } catch (err) { + // could not load index pattern, exclude it from list. + continue; + } + } + + options.push({ + label: idsAndTitles[i].title, + value: idsAndTitles[i].id, }); + + // Loading each index pattern object requires a network call so just find small number of matching index patterns + // Users can use 'searchValue' to further refine the list and locate their index pattern. + if (options.length > 15) { + break; + } + } + this.setState({ isLoading: false, options, }); - if (onNoIndexPatterns && searchValue === '' && options.length === 0) { - onNoIndexPatterns(); + if (this.props.onNoIndexPatterns && searchValue === '' && options.length === 0) { + this.props.onNoIndexPatterns(); } }, 300); diff --git a/test/functional/apps/visualize/input_control_vis/input_control_options.ts b/test/functional/apps/visualize/input_control_vis/input_control_options.ts index dc02cada9a712..2e3b5d758436e 100644 --- a/test/functional/apps/visualize/input_control_vis/input_control_options.ts +++ b/test/functional/apps/visualize/input_control_vis/input_control_options.ts @@ -31,7 +31,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { ); await PageObjects.visEditor.clickVisEditorTab('controls'); await PageObjects.visEditor.addInputControl(); - await comboBox.set('indexPatternSelect-0', 'logstash- '); + await comboBox.set('indexPatternSelect-0', 'logstash-'); await comboBox.set('fieldSelect-0', FIELD_NAME); await PageObjects.visEditor.clickGo(); }); From 012f1c199be989c41cbf599fb8fcb912a749e2d2 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Fri, 9 Apr 2021 08:57:13 +0200 Subject: [PATCH 2/8] [APM] Run precommit tasks sequentially (#96551) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- x-pack/plugins/apm/scripts/precommit.js | 32 ++++++++++++------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/x-pack/plugins/apm/scripts/precommit.js b/x-pack/plugins/apm/scripts/precommit.js index 695a9ba70f5d7..88d2e169dd542 100644 --- a/x-pack/plugins/apm/scripts/precommit.js +++ b/x-pack/plugins/apm/scripts/precommit.js @@ -28,19 +28,8 @@ const testTsconfig = resolve(root, 'x-pack/test/tsconfig.json'); const tasks = new Listr( [ { - title: 'Jest', - task: () => - execa( - 'node', - [ - resolve(__dirname, './jest.js'), - '--reporters', - resolve(__dirname, '../../../../node_modules/jest-silent-reporter'), - '--collect-coverage', - 'false', - ], - execaOpts - ), + title: 'Lint', + task: () => execa('node', [resolve(__dirname, 'eslint.js')], execaOpts), }, { title: 'Typescript', @@ -72,11 +61,22 @@ const tasks = new Listr( ), }, { - title: 'Lint', - task: () => execa('node', [resolve(__dirname, 'eslint.js')], execaOpts), + title: 'Jest', + task: () => + execa( + 'node', + [ + resolve(__dirname, './jest.js'), + '--reporters', + resolve(__dirname, '../../../../node_modules/jest-silent-reporter'), + '--collect-coverage', + 'false', + ], + execaOpts + ), }, ], - { exitOnError: true, concurrent: true } + { exitOnError: true, concurrent: false } ); tasks.run().catch((error) => { From dfaf3ac8f53126d6e481a4e5feb33d373332a9ad Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Fri, 9 Apr 2021 10:35:44 +0200 Subject: [PATCH 3/8] [RAC] Rule registry plugin (#95903) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- docs/developer/plugin-list.asciidoc | 4 + typings/elasticsearch/search.d.ts | 10 + .../apm/common/environment_filter_values.ts | 18 + x-pack/plugins/apm/kibana.json | 4 +- .../scripts/optimize-tsconfig/tsconfig.json | 1 + .../apm/server/lib/alerts/action_variables.ts | 12 +- .../server/lib/alerts/alerting_es_client.ts | 28 +- .../alerts/create_apm_lifecycle_rule_type.ts | 11 + .../server/lib/alerts/register_apm_alerts.ts | 33 +- .../register_error_count_alert_type.test.ts | 223 +- .../alerts/register_error_count_alert_type.ts | 248 +- ...egister_transaction_duration_alert_type.ts | 212 +- ...action_duration_anomaly_alert_type.test.ts | 312 +- ...transaction_duration_anomaly_alert_type.ts | 381 +- ..._transaction_error_rate_alert_type.test.ts | 303 +- ...ister_transaction_error_rate_alert_type.ts | 284 +- .../apm/server/lib/alerts/test_utils/index.ts | 64 + x-pack/plugins/apm/server/plugin.ts | 34 +- x-pack/plugins/apm/server/routes/typings.ts | 18 - x-pack/plugins/apm/server/types.ts | 2 +- x-pack/plugins/apm/tsconfig.json | 1 + .../server/es/cluster_client_adapter.test.ts | 3 +- .../server/es/cluster_client_adapter.ts | 40 +- x-pack/plugins/event_log/server/es/context.ts | 2 +- x-pack/plugins/event_log/server/index.ts | 4 + x-pack/plugins/observability/kibana.json | 2 +- x-pack/plugins/observability/server/plugin.ts | 27 +- x-pack/plugins/observability/tsconfig.json | 1 + x-pack/plugins/rule_registry/README.md | 68 + x-pack/plugins/rule_registry/common/index.ts | 8 + x-pack/plugins/rule_registry/common/types.ts | 20 + x-pack/plugins/rule_registry/jest.config.js | 12 + x-pack/plugins/rule_registry/kibana.json | 13 + .../scripts/generate_ecs_fieldmap/index.js | 81 + .../server/generated/ecs_field_map.ts | 3374 ++++++++++++++++ .../server/generated/ecs_mappings.json | 3416 +++++++++++++++++ x-pack/plugins/rule_registry/server/index.ts | 29 + x-pack/plugins/rule_registry/server/plugin.ts | 49 + .../index.ts | 174 + .../types.ts | 51 + .../rule_registry/defaults/field_map.ts | 34 + .../rule_registry/defaults/ilm_policy.ts | 28 + .../field_map/mapping_from_field_map.ts | 32 + .../field_map/merge_field_maps.ts | 50 + .../field_map/pick_with_patterns.test.ts | 71 + .../field_map/pick_with_patterns.ts | 66 + .../runtime_type_from_fieldmap.test.ts | 95 + .../field_map/runtime_type_from_fieldmap.ts | 108 + .../server/rule_registry/index.ts | 240 ++ .../create_lifecycle_rule_type_factory.ts | 235 ++ .../server/rule_registry/types.ts | 44 + x-pack/plugins/rule_registry/server/types.ts | 100 + x-pack/plugins/rule_registry/tsconfig.json | 15 + x-pack/scripts/functional_tests.js | 1 + .../test/apm_api_integration/common/config.ts | 13 +- .../test/apm_api_integration/configs/index.ts | 6 + .../test/apm_api_integration/rules/config.ts | 10 + .../tests/alerts/rule_registry.ts | 387 ++ .../test/apm_api_integration/tests/index.ts | 4 + 59 files changed, 9812 insertions(+), 1304 deletions(-) create mode 100644 x-pack/plugins/apm/server/lib/alerts/create_apm_lifecycle_rule_type.ts create mode 100644 x-pack/plugins/apm/server/lib/alerts/test_utils/index.ts create mode 100644 x-pack/plugins/rule_registry/README.md create mode 100644 x-pack/plugins/rule_registry/common/index.ts create mode 100644 x-pack/plugins/rule_registry/common/types.ts create mode 100644 x-pack/plugins/rule_registry/jest.config.js create mode 100644 x-pack/plugins/rule_registry/kibana.json create mode 100644 x-pack/plugins/rule_registry/scripts/generate_ecs_fieldmap/index.js create mode 100644 x-pack/plugins/rule_registry/server/generated/ecs_field_map.ts create mode 100644 x-pack/plugins/rule_registry/server/generated/ecs_mappings.json create mode 100644 x-pack/plugins/rule_registry/server/index.ts create mode 100644 x-pack/plugins/rule_registry/server/plugin.ts create mode 100644 x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/index.ts create mode 100644 x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/types.ts create mode 100644 x-pack/plugins/rule_registry/server/rule_registry/defaults/field_map.ts create mode 100644 x-pack/plugins/rule_registry/server/rule_registry/defaults/ilm_policy.ts create mode 100644 x-pack/plugins/rule_registry/server/rule_registry/field_map/mapping_from_field_map.ts create mode 100644 x-pack/plugins/rule_registry/server/rule_registry/field_map/merge_field_maps.ts create mode 100644 x-pack/plugins/rule_registry/server/rule_registry/field_map/pick_with_patterns.test.ts create mode 100644 x-pack/plugins/rule_registry/server/rule_registry/field_map/pick_with_patterns.ts create mode 100644 x-pack/plugins/rule_registry/server/rule_registry/field_map/runtime_type_from_fieldmap.test.ts create mode 100644 x-pack/plugins/rule_registry/server/rule_registry/field_map/runtime_type_from_fieldmap.ts create mode 100644 x-pack/plugins/rule_registry/server/rule_registry/index.ts create mode 100644 x-pack/plugins/rule_registry/server/rule_registry/rule_type_helpers/create_lifecycle_rule_type_factory.ts create mode 100644 x-pack/plugins/rule_registry/server/rule_registry/types.ts create mode 100644 x-pack/plugins/rule_registry/server/types.ts create mode 100644 x-pack/plugins/rule_registry/tsconfig.json create mode 100644 x-pack/test/apm_api_integration/rules/config.ts create mode 100644 x-pack/test/apm_api_integration/tests/alerts/rule_registry.ts diff --git a/docs/developer/plugin-list.asciidoc b/docs/developer/plugin-list.asciidoc index 691d7fb82f3bc..0c40c2a8c4db9 100644 --- a/docs/developer/plugin-list.asciidoc +++ b/docs/developer/plugin-list.asciidoc @@ -485,6 +485,10 @@ Elastic. |Welcome to the Kibana rollup plugin! This plugin provides Kibana support for Elasticsearch's rollup feature. Please refer to the Elasticsearch documentation to understand rollup indices and how to create rollup jobs. +|{kib-repo}blob/{branch}/x-pack/plugins/rule_registry/README.md[ruleRegistry] +|The rule registry plugin aims to make it easy for rule type producers to have their rules produce the data that they need to build rich experiences on top of a unified experience, without the risk of mapping conflicts. + + |{kib-repo}blob/{branch}/x-pack/plugins/runtime_fields/README.md[runtimeFields] |Welcome to the home of the runtime field editor and everything related to runtime fields! diff --git a/typings/elasticsearch/search.d.ts b/typings/elasticsearch/search.d.ts index fce08df1c0a04..c9bf3b1d8b7bc 100644 --- a/typings/elasticsearch/search.d.ts +++ b/typings/elasticsearch/search.d.ts @@ -370,6 +370,16 @@ export type AggregateOf< missing: { doc_count: number; } & SubAggregateOf; + multi_terms: { + doc_count_error_upper_bound: number; + sum_other_doc_count: number; + buckets: Array< + { + doc_count: number; + key: string[]; + } & SubAggregateOf + >; + }; nested: { doc_count: number; } & SubAggregateOf; diff --git a/x-pack/plugins/apm/common/environment_filter_values.ts b/x-pack/plugins/apm/common/environment_filter_values.ts index e091b53b2e5b8..c80541ee1ba6b 100644 --- a/x-pack/plugins/apm/common/environment_filter_values.ts +++ b/x-pack/plugins/apm/common/environment_filter_values.ts @@ -22,11 +22,13 @@ const environmentLabels: Record = { }; export const ENVIRONMENT_ALL = { + esFieldValue: undefined, value: ENVIRONMENT_ALL_VALUE, text: environmentLabels[ENVIRONMENT_ALL_VALUE], }; export const ENVIRONMENT_NOT_DEFINED = { + esFieldValue: undefined, value: ENVIRONMENT_NOT_DEFINED_VALUE, text: environmentLabels[ENVIRONMENT_NOT_DEFINED_VALUE], }; @@ -35,6 +37,22 @@ export function getEnvironmentLabel(environment: string) { return environmentLabels[environment] || environment; } +export function parseEnvironmentUrlParam(environment: string) { + if (environment === ENVIRONMENT_ALL_VALUE) { + return ENVIRONMENT_ALL; + } + + if (environment === ENVIRONMENT_NOT_DEFINED_VALUE) { + return ENVIRONMENT_NOT_DEFINED; + } + + return { + esFieldValue: environment, + value: environment, + text: environment, + }; +} + // returns the environment url param that should be used // based on the requested environment. If the requested // environment is different from the URL parameter, we'll diff --git a/x-pack/plugins/apm/kibana.json b/x-pack/plugins/apm/kibana.json index e340f8bf19126..28e4a7b36e740 100644 --- a/x-pack/plugins/apm/kibana.json +++ b/x-pack/plugins/apm/kibana.json @@ -9,7 +9,8 @@ "licensing", "triggersActionsUi", "embeddable", - "infra" + "infra", + "observability" ], "optionalPlugins": [ "spaces", @@ -18,7 +19,6 @@ "taskManager", "actions", "alerting", - "observability", "security", "ml", "home", diff --git a/x-pack/plugins/apm/scripts/optimize-tsconfig/tsconfig.json b/x-pack/plugins/apm/scripts/optimize-tsconfig/tsconfig.json index 319eb53313231..40d42298b967b 100644 --- a/x-pack/plugins/apm/scripts/optimize-tsconfig/tsconfig.json +++ b/x-pack/plugins/apm/scripts/optimize-tsconfig/tsconfig.json @@ -2,6 +2,7 @@ "include": [ "./x-pack/plugins/apm/**/*", "./x-pack/plugins/observability/**/*", + "./x-pack/plugins/rule_registry/**/*", "./typings/**/*" ], "exclude": [ diff --git a/x-pack/plugins/apm/server/lib/alerts/action_variables.ts b/x-pack/plugins/apm/server/lib/alerts/action_variables.ts index 473912c4177a9..b065da7123dec 100644 --- a/x-pack/plugins/apm/server/lib/alerts/action_variables.ts +++ b/x-pack/plugins/apm/server/lib/alerts/action_variables.ts @@ -13,28 +13,28 @@ export const apmActionVariables = { 'xpack.apm.alerts.action_variables.serviceName', { defaultMessage: 'The service the alert is created for' } ), - name: 'serviceName', + name: 'serviceName' as const, }, transactionType: { description: i18n.translate( 'xpack.apm.alerts.action_variables.transactionType', { defaultMessage: 'The transaction type the alert is created for' } ), - name: 'transactionType', + name: 'transactionType' as const, }, environment: { description: i18n.translate( 'xpack.apm.alerts.action_variables.environment', { defaultMessage: 'The transaction type the alert is created for' } ), - name: 'environment', + name: 'environment' as const, }, threshold: { description: i18n.translate('xpack.apm.alerts.action_variables.threshold', { defaultMessage: 'Any trigger value above this value will cause the alert to fire', }), - name: 'threshold', + name: 'threshold' as const, }, triggerValue: { description: i18n.translate( @@ -44,7 +44,7 @@ export const apmActionVariables = { 'The value that breached the threshold and triggered the alert', } ), - name: 'triggerValue', + name: 'triggerValue' as const, }, interval: { description: i18n.translate( @@ -54,6 +54,6 @@ export const apmActionVariables = { 'The length and unit of the time period where the alert conditions were met', } ), - name: 'interval', + name: 'interval' as const, }, }; diff --git a/x-pack/plugins/apm/server/lib/alerts/alerting_es_client.ts b/x-pack/plugins/apm/server/lib/alerts/alerting_es_client.ts index 9a0ba514bb479..e3d5e5481caa5 100644 --- a/x-pack/plugins/apm/server/lib/alerts/alerting_es_client.ts +++ b/x-pack/plugins/apm/server/lib/alerts/alerting_es_client.ts @@ -5,28 +5,24 @@ * 2.0. */ -import { ApiResponse } from '@elastic/elasticsearch'; -import { ThresholdMetActionGroupId } from '../../../common/alert_types'; import { ESSearchRequest, ESSearchResponse, } from '../../../../../../typings/elasticsearch'; -import { - AlertInstanceContext, - AlertInstanceState, - AlertServices, -} from '../../../../alerting/server'; +import { AlertServices } from '../../../../alerting/server'; -export function alertingEsClient( - services: AlertServices< - AlertInstanceState, - AlertInstanceContext, - ThresholdMetActionGroupId - >, +export async function alertingEsClient( + scopedClusterClient: AlertServices< + never, + never, + never + >['scopedClusterClient'], params: TParams -): Promise>> { - return (services.scopedClusterClient.asCurrentUser.search({ +): Promise> { + const response = await scopedClusterClient.asCurrentUser.search({ ...params, ignore_unavailable: true, - }) as unknown) as Promise>>; + }); + + return (response.body as unknown) as ESSearchResponse; } diff --git a/x-pack/plugins/apm/server/lib/alerts/create_apm_lifecycle_rule_type.ts b/x-pack/plugins/apm/server/lib/alerts/create_apm_lifecycle_rule_type.ts new file mode 100644 index 0000000000000..8d250a5765cce --- /dev/null +++ b/x-pack/plugins/apm/server/lib/alerts/create_apm_lifecycle_rule_type.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createLifecycleRuleTypeFactory } from '../../../../rule_registry/server'; +import { APMRuleRegistry } from '../../plugin'; + +export const createAPMLifecycleRuleType = createLifecycleRuleTypeFactory(); diff --git a/x-pack/plugins/apm/server/lib/alerts/register_apm_alerts.ts b/x-pack/plugins/apm/server/lib/alerts/register_apm_alerts.ts index a9824c130faa5..9a362efa90ac0 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_apm_alerts.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_apm_alerts.ts @@ -6,38 +6,25 @@ */ import { Observable } from 'rxjs'; -import { AlertingPlugin } from '../../../../alerting/server'; -import { ActionsPlugin } from '../../../../actions/server'; +import { Logger } from 'kibana/server'; import { registerTransactionDurationAlertType } from './register_transaction_duration_alert_type'; import { registerTransactionDurationAnomalyAlertType } from './register_transaction_duration_anomaly_alert_type'; import { registerErrorCountAlertType } from './register_error_count_alert_type'; import { APMConfig } from '../..'; import { MlPluginSetup } from '../../../../ml/server'; import { registerTransactionErrorRateAlertType } from './register_transaction_error_rate_alert_type'; +import { APMRuleRegistry } from '../../plugin'; -interface Params { - alerting: AlertingPlugin['setup']; - actions: ActionsPlugin['setup']; +export interface RegisterRuleDependencies { + registry: APMRuleRegistry; ml?: MlPluginSetup; config$: Observable; + logger: Logger; } -export function registerApmAlerts(params: Params) { - registerTransactionDurationAlertType({ - alerting: params.alerting, - config$: params.config$, - }); - registerTransactionDurationAnomalyAlertType({ - alerting: params.alerting, - ml: params.ml, - config$: params.config$, - }); - registerErrorCountAlertType({ - alerting: params.alerting, - config$: params.config$, - }); - registerTransactionErrorRateAlertType({ - alerting: params.alerting, - config$: params.config$, - }); +export function registerApmAlerts(dependencies: RegisterRuleDependencies) { + registerTransactionDurationAlertType(dependencies); + registerTransactionDurationAnomalyAlertType(dependencies); + registerErrorCountAlertType(dependencies); + registerTransactionErrorRateAlertType(dependencies); } diff --git a/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.test.ts b/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.test.ts index d7dd7aee3ca25..5758dea1860b2 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.test.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.test.ts @@ -5,50 +5,17 @@ * 2.0. */ -import { Observable } from 'rxjs'; -import * as Rx from 'rxjs'; -import { toArray, map } from 'rxjs/operators'; - -import { AlertingPlugin } from '../../../../alerting/server'; -import { APMConfig } from '../..'; - import { registerErrorCountAlertType } from './register_error_count_alert_type'; -import { elasticsearchServiceMock } from 'src/core/server/mocks'; // eslint-disable-next-line @kbn/eslint/no-restricted-paths import { elasticsearchClientMock } from 'src/core/server/elasticsearch/client/mocks'; - -type Operator = (source: Rx.Observable) => Rx.Observable; -const pipeClosure = (fn: Operator): Operator => { - return (source: Rx.Observable) => { - return Rx.defer(() => fn(source)); - }; -}; -const mockedConfig$ = (Rx.of('apm_oss.errorIndices').pipe( - pipeClosure((source$) => { - return source$.pipe(map((i) => i)); - }), - toArray() -) as unknown) as Observable; +import { createRuleTypeMocks } from './test_utils'; describe('Error count alert', () => { it("doesn't send an alert when error count is less than threshold", async () => { - let alertExecutor: any; - const alerting = { - registerType: ({ executor }) => { - alertExecutor = executor; - }, - } as AlertingPlugin['setup']; + const { services, dependencies, executor } = createRuleTypeMocks(); - registerErrorCountAlertType({ - alerting, - config$: mockedConfig$, - }); - expect(alertExecutor).toBeDefined(); + registerErrorCountAlertType(dependencies); - const services = { - scopedClusterClient: elasticsearchServiceMock.createScopedClusterClient(), - alertInstanceFactory: jest.fn(), - }; const params = { threshold: 1 }; services.scopedClusterClient.asCurrentUser.search.mockReturnValue( @@ -71,30 +38,21 @@ describe('Error count alert', () => { }) ); - await alertExecutor!({ services, params }); + await executor({ params }); expect(services.alertInstanceFactory).not.toBeCalled(); }); - it('sends alerts with service name and environment', async () => { - let alertExecutor: any; - const alerting = { - registerType: ({ executor }) => { - alertExecutor = executor; - }, - } as AlertingPlugin['setup']; + it('sends alerts with service name and environment for those that exceeded the threshold', async () => { + const { + services, + dependencies, + executor, + scheduleActions, + } = createRuleTypeMocks(); - registerErrorCountAlertType({ - alerting, - config$: mockedConfig$, - }); - expect(alertExecutor).toBeDefined(); + registerErrorCountAlertType(dependencies); - const scheduleActions = jest.fn(); - const services = { - scopedClusterClient: elasticsearchServiceMock.createScopedClusterClient(), - alertInstanceFactory: jest.fn(() => ({ scheduleActions })), - }; - const params = { threshold: 1, windowSize: 5, windowUnit: 'm' }; + const params = { threshold: 2, windowSize: 5, windowUnit: 'm' }; services.scopedClusterClient.asCurrentUser.search.mockReturnValue( elasticsearchClientMock.createSuccessTransportRequestPromise({ @@ -106,18 +64,62 @@ describe('Error count alert', () => { }, }, aggregations: { - services: { + error_counts: { buckets: [ { - key: 'foo', - environments: { - buckets: [{ key: 'env-foo' }, { key: 'env-foo-2' }], + key: ['foo', 'env-foo'], + doc_count: 5, + latest: { + top: [ + { + metrics: { + 'service.name': 'foo', + 'service.environment': 'env-foo', + }, + }, + ], + }, + }, + { + key: ['foo', 'env-foo-2'], + doc_count: 4, + latest: { + top: [ + { + metrics: { + 'service.name': 'foo', + 'service.environment': 'env-foo-2', + }, + }, + ], }, }, { - key: 'bar', - environments: { - buckets: [{ key: 'env-bar' }, { key: 'env-bar-2' }], + key: ['bar', 'env-bar'], + doc_count: 3, + latest: { + top: [ + { + metrics: { + 'service.name': 'bar', + 'service.environment': 'env-bar', + }, + }, + ], + }, + }, + { + key: ['bar', 'env-bar-2'], + doc_count: 1, + latest: { + top: [ + { + metrics: { + 'service.name': 'bar', + 'service.environment': 'env-bar-2', + }, + }, + ], }, }, ], @@ -134,115 +136,36 @@ describe('Error count alert', () => { }) ); - await alertExecutor!({ services, params }); + await executor({ params }); [ 'apm.error_rate_foo_env-foo', 'apm.error_rate_foo_env-foo-2', 'apm.error_rate_bar_env-bar', - 'apm.error_rate_bar_env-bar-2', ].forEach((instanceName) => expect(services.alertInstanceFactory).toHaveBeenCalledWith(instanceName) ); + expect(scheduleActions).toHaveBeenCalledTimes(3); + expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { serviceName: 'foo', environment: 'env-foo', - threshold: 1, - triggerValue: 2, + threshold: 2, + triggerValue: 5, interval: '5m', }); expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { serviceName: 'foo', environment: 'env-foo-2', - threshold: 1, - triggerValue: 2, + threshold: 2, + triggerValue: 4, interval: '5m', }); expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { serviceName: 'bar', environment: 'env-bar', - threshold: 1, - triggerValue: 2, - interval: '5m', - }); - expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { - serviceName: 'bar', - environment: 'env-bar-2', - threshold: 1, - triggerValue: 2, - interval: '5m', - }); - }); - it('sends alerts with service name', async () => { - let alertExecutor: any; - const alerting = { - registerType: ({ executor }) => { - alertExecutor = executor; - }, - } as AlertingPlugin['setup']; - - registerErrorCountAlertType({ - alerting, - config$: mockedConfig$, - }); - expect(alertExecutor).toBeDefined(); - - const scheduleActions = jest.fn(); - const services = { - scopedClusterClient: elasticsearchServiceMock.createScopedClusterClient(), - alertInstanceFactory: jest.fn(() => ({ scheduleActions })), - }; - const params = { threshold: 1, windowSize: 5, windowUnit: 'm' }; - - services.scopedClusterClient.asCurrentUser.search.mockReturnValue( - elasticsearchClientMock.createSuccessTransportRequestPromise({ - hits: { - hits: [], - total: { - relation: 'eq', - value: 2, - }, - }, - aggregations: { - services: { - buckets: [ - { - key: 'foo', - }, - { - key: 'bar', - }, - ], - }, - }, - took: 0, - timed_out: false, - _shards: { - failed: 0, - skipped: 0, - successful: 1, - total: 1, - }, - }) - ); - - await alertExecutor!({ services, params }); - ['apm.error_rate_foo', 'apm.error_rate_bar'].forEach((instanceName) => - expect(services.alertInstanceFactory).toHaveBeenCalledWith(instanceName) - ); - - expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { - serviceName: 'foo', - environment: undefined, - threshold: 1, - triggerValue: 2, - interval: '5m', - }); - expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { - serviceName: 'bar', - environment: undefined, - threshold: 1, - triggerValue: 2, + threshold: 2, + triggerValue: 3, interval: '5m', }); }); diff --git a/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.ts b/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.ts index 0120891a8f868..8240e0c369d1f 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.ts @@ -5,22 +5,11 @@ * 2.0. */ -import { schema, TypeOf } from '@kbn/config-schema'; -import { isEmpty } from 'lodash'; -import { Observable } from 'rxjs'; +import { schema } from '@kbn/config-schema'; import { take } from 'rxjs/operators'; -import { APMConfig } from '../..'; -import { - AlertingPlugin, - AlertInstanceContext, - AlertInstanceState, - AlertTypeState, -} from '../../../../alerting/server'; -import { - AlertType, - ALERT_TYPES_CONFIG, - ThresholdMetActionGroupId, -} from '../../../common/alert_types'; +import { ENVIRONMENT_NOT_DEFINED } from '../../../common/environment_filter_values'; +import { asMutableArray } from '../../../common/utils/as_mutable_array'; +import { AlertType, ALERT_TYPES_CONFIG } from '../../../common/alert_types'; import { PROCESSOR_EVENT, SERVICE_ENVIRONMENT, @@ -31,11 +20,8 @@ import { environmentQuery } from '../../../server/utils/queries'; import { getApmIndices } from '../settings/apm_indices/get_apm_indices'; import { apmActionVariables } from './action_variables'; import { alertingEsClient } from './alerting_es_client'; - -interface RegisterAlertParams { - alerting: AlertingPlugin['setup']; - config$: Observable; -} +import { RegisterRuleDependencies } from './register_apm_alerts'; +import { createAPMLifecycleRuleType } from './create_apm_lifecycle_rule_type'; const paramsSchema = schema.object({ windowSize: schema.number(), @@ -48,127 +34,131 @@ const paramsSchema = schema.object({ const alertTypeConfig = ALERT_TYPES_CONFIG[AlertType.ErrorCount]; export function registerErrorCountAlertType({ - alerting, + registry, config$, -}: RegisterAlertParams) { - alerting.registerType< - TypeOf, - AlertTypeState, - AlertInstanceState, - AlertInstanceContext, - ThresholdMetActionGroupId - >({ - id: AlertType.ErrorCount, - name: alertTypeConfig.name, - actionGroups: alertTypeConfig.actionGroups, - defaultActionGroupId: alertTypeConfig.defaultActionGroupId, - validate: { - params: paramsSchema, - }, - actionVariables: { - context: [ - apmActionVariables.serviceName, - apmActionVariables.environment, - apmActionVariables.threshold, - apmActionVariables.triggerValue, - apmActionVariables.interval, - ], - }, - producer: 'apm', - minimumLicenseRequired: 'basic', - executor: async ({ services, params }) => { - const config = await config$.pipe(take(1)).toPromise(); - const alertParams = params; - const indices = await getApmIndices({ - config, - savedObjectsClient: services.savedObjectsClient, - }); - const maxServiceEnvironments = config['xpack.apm.maxServiceEnvironments']; +}: RegisterRuleDependencies) { + registry.registerType( + createAPMLifecycleRuleType({ + id: AlertType.ErrorCount, + name: alertTypeConfig.name, + actionGroups: alertTypeConfig.actionGroups, + defaultActionGroupId: alertTypeConfig.defaultActionGroupId, + validate: { + params: paramsSchema, + }, + actionVariables: { + context: [ + apmActionVariables.serviceName, + apmActionVariables.environment, + apmActionVariables.threshold, + apmActionVariables.triggerValue, + apmActionVariables.interval, + ], + }, + producer: 'apm', + minimumLicenseRequired: 'basic', + executor: async ({ services, params }) => { + const config = await config$.pipe(take(1)).toPromise(); + const alertParams = params; + const indices = await getApmIndices({ + config, + savedObjectsClient: services.savedObjectsClient, + }); - const searchParams = { - index: indices['apm_oss.errorIndices'], - size: 0, - body: { - track_total_hits: true, - query: { - bool: { - filter: [ - { - range: { - '@timestamp': { - gte: `now-${alertParams.windowSize}${alertParams.windowUnit}`, + const searchParams = { + index: indices['apm_oss.errorIndices'], + size: 0, + body: { + query: { + bool: { + filter: [ + { + range: { + '@timestamp': { + gte: `now-${alertParams.windowSize}${alertParams.windowUnit}`, + }, }, }, - }, - { term: { [PROCESSOR_EVENT]: ProcessorEvent.error } }, - ...(alertParams.serviceName - ? [{ term: { [SERVICE_NAME]: alertParams.serviceName } }] - : []), - ...environmentQuery(alertParams.environment), - ], - }, - }, - aggs: { - services: { - terms: { - field: SERVICE_NAME, - size: 50, + { term: { [PROCESSOR_EVENT]: ProcessorEvent.error } }, + ...(alertParams.serviceName + ? [{ term: { [SERVICE_NAME]: alertParams.serviceName } }] + : []), + ...environmentQuery(alertParams.environment), + ], }, - aggs: { - environments: { - terms: { - field: SERVICE_ENVIRONMENT, - size: maxServiceEnvironments, + }, + aggs: { + error_counts: { + multi_terms: { + terms: [ + { field: SERVICE_NAME }, + { field: SERVICE_ENVIRONMENT, missing: '' }, + ], + size: 10000, + }, + aggs: { + latest: { + top_metrics: { + metrics: asMutableArray([ + { field: SERVICE_NAME }, + { field: SERVICE_ENVIRONMENT }, + ] as const), + sort: { + '@timestamp': 'desc' as const, + }, + }, }, }, }, }, }, - }, - }; + }; + + const response = await alertingEsClient( + services.scopedClusterClient, + searchParams + ); - const { body: response } = await alertingEsClient(services, searchParams); - const errorCount = response.hits.total.value; + const errorCountResults = + response.aggregations?.error_counts.buckets.map((bucket) => { + const latest = bucket.latest.top[0].metrics; - if (errorCount > alertParams.threshold) { - function scheduleAction({ - serviceName, - environment, - }: { - serviceName: string; - environment?: string; - }) { - const alertInstanceName = [ - AlertType.ErrorCount, - serviceName, - environment, - ] - .filter((name) => name) - .join('_'); + return { + serviceName: latest['service.name'] as string, + environment: latest['service.environment'] as string | undefined, + errorCount: bucket.doc_count, + }; + }) ?? []; - const alertInstance = services.alertInstanceFactory( - alertInstanceName - ); - alertInstance.scheduleActions(alertTypeConfig.defaultActionGroupId, { - serviceName, - environment, - threshold: alertParams.threshold, - triggerValue: errorCount, - interval: `${alertParams.windowSize}${alertParams.windowUnit}`, + errorCountResults + .filter((result) => result.errorCount >= alertParams.threshold) + .forEach((result) => { + const { serviceName, environment, errorCount } = result; + + services + .alertWithLifecycle({ + id: [AlertType.ErrorCount, serviceName, environment] + .filter((name) => name) + .join('_'), + fields: { + [SERVICE_NAME]: serviceName, + ...(environment + ? { [SERVICE_ENVIRONMENT]: environment } + : {}), + [PROCESSOR_EVENT]: 'error', + }, + }) + .scheduleActions(alertTypeConfig.defaultActionGroupId, { + serviceName, + environment: environment || ENVIRONMENT_NOT_DEFINED.text, + threshold: alertParams.threshold, + triggerValue: errorCount, + interval: `${alertParams.windowSize}${alertParams.windowUnit}`, + }); }); - } - response.aggregations?.services.buckets.forEach((serviceBucket) => { - const serviceName = serviceBucket.key as string; - if (isEmpty(serviceBucket.environments?.buckets)) { - scheduleAction({ serviceName }); - } else { - serviceBucket.environments.buckets.forEach((envBucket) => { - const environment = envBucket.key as string; - scheduleAction({ serviceName, environment }); - }); - } - }); - } - }, - }); + + return {}; + }, + }) + ); } diff --git a/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_alert_type.ts b/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_alert_type.ts index 500e0744d5638..6ca1c4370d6ae 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_alert_type.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_alert_type.ts @@ -6,10 +6,9 @@ */ import { schema } from '@kbn/config-schema'; -import { Observable } from 'rxjs'; import { take } from 'rxjs/operators'; -import { APMConfig } from '../..'; -import { AlertingPlugin } from '../../../../alerting/server'; +import { QueryContainer } from '@elastic/elasticsearch/api/types'; +import { parseEnvironmentUrlParam } from '../../../common/environment_filter_values'; import { AlertType, ALERT_TYPES_CONFIG } from '../../../common/alert_types'; import { PROCESSOR_EVENT, @@ -24,11 +23,8 @@ import { environmentQuery } from '../../../server/utils/queries'; import { getApmIndices } from '../settings/apm_indices/get_apm_indices'; import { apmActionVariables } from './action_variables'; import { alertingEsClient } from './alerting_es_client'; - -interface RegisterAlertParams { - alerting: AlertingPlugin['setup']; - config$: Observable; -} +import { RegisterRuleDependencies } from './register_apm_alerts'; +import { createAPMLifecycleRuleType } from './create_apm_lifecycle_rule_type'; const paramsSchema = schema.object({ serviceName: schema.string(), @@ -47,116 +43,126 @@ const paramsSchema = schema.object({ const alertTypeConfig = ALERT_TYPES_CONFIG[AlertType.TransactionDuration]; export function registerTransactionDurationAlertType({ - alerting, + registry, config$, -}: RegisterAlertParams) { - alerting.registerType({ - id: AlertType.TransactionDuration, - name: alertTypeConfig.name, - actionGroups: alertTypeConfig.actionGroups, - defaultActionGroupId: alertTypeConfig.defaultActionGroupId, - validate: { - params: paramsSchema, - }, - actionVariables: { - context: [ - apmActionVariables.serviceName, - apmActionVariables.transactionType, - apmActionVariables.environment, - apmActionVariables.threshold, - apmActionVariables.triggerValue, - apmActionVariables.interval, - ], - }, - producer: 'apm', - minimumLicenseRequired: 'basic', - executor: async ({ services, params }) => { - const config = await config$.pipe(take(1)).toPromise(); - const alertParams = params; - const indices = await getApmIndices({ - config, - savedObjectsClient: services.savedObjectsClient, - }); - const maxServiceEnvironments = config['xpack.apm.maxServiceEnvironments']; +}: RegisterRuleDependencies) { + registry.registerType( + createAPMLifecycleRuleType({ + id: AlertType.TransactionDuration, + name: alertTypeConfig.name, + actionGroups: alertTypeConfig.actionGroups, + defaultActionGroupId: alertTypeConfig.defaultActionGroupId, + validate: { + params: paramsSchema, + }, + actionVariables: { + context: [ + apmActionVariables.serviceName, + apmActionVariables.transactionType, + apmActionVariables.environment, + apmActionVariables.threshold, + apmActionVariables.triggerValue, + apmActionVariables.interval, + ], + }, + producer: 'apm', + minimumLicenseRequired: 'basic', + executor: async ({ services, params }) => { + const config = await config$.pipe(take(1)).toPromise(); + const alertParams = params; + const indices = await getApmIndices({ + config, + savedObjectsClient: services.savedObjectsClient, + }); - const searchParams = { - index: indices['apm_oss.transactionIndices'], - size: 0, - body: { - query: { - bool: { - filter: [ - { - range: { - '@timestamp': { - gte: `now-${alertParams.windowSize}${alertParams.windowUnit}`, + const searchParams = { + index: indices['apm_oss.transactionIndices'], + size: 0, + body: { + query: { + bool: { + filter: [ + { + range: { + '@timestamp': { + gte: `now-${alertParams.windowSize}${alertParams.windowUnit}`, + }, }, }, - }, - { term: { [PROCESSOR_EVENT]: ProcessorEvent.transaction } }, - { term: { [SERVICE_NAME]: alertParams.serviceName } }, - { term: { [TRANSACTION_TYPE]: alertParams.transactionType } }, - ...environmentQuery(alertParams.environment), - ], + { term: { [PROCESSOR_EVENT]: ProcessorEvent.transaction } }, + { term: { [SERVICE_NAME]: alertParams.serviceName } }, + { term: { [TRANSACTION_TYPE]: alertParams.transactionType } }, + ...environmentQuery(alertParams.environment), + ] as QueryContainer[], + }, }, - }, - aggs: { - agg: - alertParams.aggregationType === 'avg' - ? { avg: { field: TRANSACTION_DURATION } } - : { - percentiles: { - field: TRANSACTION_DURATION, - percents: [ - alertParams.aggregationType === '95th' ? 95 : 99, - ], + aggs: { + latency: + alertParams.aggregationType === 'avg' + ? { avg: { field: TRANSACTION_DURATION } } + : { + percentiles: { + field: TRANSACTION_DURATION, + percents: [ + alertParams.aggregationType === '95th' ? 95 : 99, + ], + }, }, - }, - environments: { - terms: { - field: SERVICE_ENVIRONMENT, - size: maxServiceEnvironments, - }, }, }, - }, - }; + }; - const { body: response } = await alertingEsClient(services, searchParams); + const response = await alertingEsClient( + services.scopedClusterClient, + searchParams + ); - if (!response.aggregations) { - return; - } + if (!response.aggregations) { + return {}; + } - const { agg, environments } = response.aggregations; + const { latency } = response.aggregations; - const transactionDuration = - 'values' in agg ? Object.values(agg.values)[0] : agg?.value; + const transactionDuration = + 'values' in latency + ? Object.values(latency.values)[0] + : latency?.value; - const threshold = alertParams.threshold * 1000; + const threshold = alertParams.threshold * 1000; - if (transactionDuration && transactionDuration > threshold) { - const durationFormatter = getDurationFormatter(transactionDuration); - const transactionDurationFormatted = durationFormatter( - transactionDuration - ).formatted; + if (transactionDuration && transactionDuration > threshold) { + const durationFormatter = getDurationFormatter(transactionDuration); + const transactionDurationFormatted = durationFormatter( + transactionDuration + ).formatted; - environments.buckets.map((bucket) => { - const environment = bucket.key; - const alertInstance = services.alertInstanceFactory( - `${AlertType.TransactionDuration}_${environment}` + const environmentParsed = parseEnvironmentUrlParam( + alertParams.environment ); - alertInstance.scheduleActions(alertTypeConfig.defaultActionGroupId, { - transactionType: alertParams.transactionType, - serviceName: alertParams.serviceName, - environment, - threshold, - triggerValue: transactionDurationFormatted, - interval: `${alertParams.windowSize}${alertParams.windowUnit}`, - }); - }); - } - }, - }); + services + .alertWithLifecycle({ + id: `${AlertType.TransactionDuration}_${environmentParsed.text}`, + fields: { + [SERVICE_NAME]: alertParams.serviceName, + ...(environmentParsed.esFieldValue + ? { [SERVICE_ENVIRONMENT]: environmentParsed.esFieldValue } + : {}), + [TRANSACTION_TYPE]: alertParams.transactionType, + }, + }) + .scheduleActions(alertTypeConfig.defaultActionGroupId, { + transactionType: alertParams.transactionType, + serviceName: alertParams.serviceName, + environment: environmentParsed.text, + threshold, + triggerValue: transactionDurationFormatted, + interval: `${alertParams.windowSize}${alertParams.windowUnit}`, + }); + } + + return {}; + }, + }) + ); } diff --git a/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.test.ts b/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.test.ts index 5f6c07cae4b8f..b9346b2bf4649 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.test.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.test.ts @@ -4,29 +4,11 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - -import { Observable } from 'rxjs'; -import * as Rx from 'rxjs'; -import { toArray, map } from 'rxjs/operators'; -import { AlertingPlugin } from '../../../../alerting/server'; import { registerTransactionDurationAnomalyAlertType } from './register_transaction_duration_anomaly_alert_type'; -import { APMConfig } from '../..'; import { ANOMALY_SEVERITY } from '../../../../ml/common'; import { Job, MlPluginSetup } from '../../../../ml/server'; import * as GetServiceAnomalies from '../service_map/get_service_anomalies'; - -type Operator = (source: Rx.Observable) => Rx.Observable; -const pipeClosure = (fn: Operator): Operator => { - return (source: Rx.Observable) => { - return Rx.defer(() => fn(source)); - }; -}; -const mockedConfig$ = (Rx.of('apm_oss.errorIndices').pipe( - pipeClosure((source$) => { - return source$.pipe(map((i) => i)); - }), - toArray() -) as unknown) as Observable; +import { createRuleTypeMocks } from './test_utils'; describe('Transaction duration anomaly alert', () => { afterEach(() => { @@ -34,28 +16,21 @@ describe('Transaction duration anomaly alert', () => { }); describe("doesn't send alert", () => { it('ml is not defined', async () => { - let alertExecutor: any; - const alerting = { - registerType: ({ executor }) => { - alertExecutor = executor; - }, - } as AlertingPlugin['setup']; + const { services, dependencies, executor } = createRuleTypeMocks(); registerTransactionDurationAnomalyAlertType({ - alerting, + ...dependencies, ml: undefined, - config$: mockedConfig$, }); - expect(alertExecutor).toBeDefined(); - const services = { - callCluster: jest.fn(), - alertInstanceFactory: jest.fn(), - }; const params = { anomalySeverityType: ANOMALY_SEVERITY.MINOR }; - await alertExecutor!({ services, params }); - expect(services.callCluster).not.toHaveBeenCalled(); + await executor({ params }); + + expect( + services.scopedClusterClient.asCurrentUser.search + ).not.toHaveBeenCalled(); + expect(services.alertInstanceFactory).not.toHaveBeenCalled(); }); @@ -64,13 +39,7 @@ describe('Transaction duration anomaly alert', () => { .spyOn(GetServiceAnomalies, 'getMLJobs') .mockReturnValue(Promise.resolve([])); - let alertExecutor: any; - - const alerting = { - registerType: ({ executor }) => { - alertExecutor = executor; - }, - } as AlertingPlugin['setup']; + const { services, dependencies, executor } = createRuleTypeMocks(); const ml = ({ mlSystemProvider: () => ({ mlAnomalySearch: jest.fn() }), @@ -78,117 +47,47 @@ describe('Transaction duration anomaly alert', () => { } as unknown) as MlPluginSetup; registerTransactionDurationAnomalyAlertType({ - alerting, + ...dependencies, ml, - config$: mockedConfig$, }); - expect(alertExecutor).toBeDefined(); - const services = { - callCluster: jest.fn(), - alertInstanceFactory: jest.fn(), - }; const params = { anomalySeverityType: ANOMALY_SEVERITY.MINOR }; - await alertExecutor!({ services, params }); - expect(services.callCluster).not.toHaveBeenCalled(); - expect(services.alertInstanceFactory).not.toHaveBeenCalled(); - }); + await executor({ params }); + expect( + services.scopedClusterClient.asCurrentUser.search + ).not.toHaveBeenCalled(); - it('anomaly is less than threshold', async () => { - jest - .spyOn(GetServiceAnomalies, 'getMLJobs') - .mockReturnValue( - Promise.resolve([{ job_id: '1' }, { job_id: '2' }] as Job[]) - ); - - let alertExecutor: any; - - const alerting = { - registerType: ({ executor }) => { - alertExecutor = executor; - }, - } as AlertingPlugin['setup']; - - const ml = ({ - mlSystemProvider: () => ({ - mlAnomalySearch: () => ({ - hits: { total: { value: 0 } }, - }), - }), - anomalyDetectorsProvider: jest.fn(), - } as unknown) as MlPluginSetup; - - registerTransactionDurationAnomalyAlertType({ - alerting, - ml, - config$: mockedConfig$, - }); - expect(alertExecutor).toBeDefined(); - - const services = { - callCluster: jest.fn(), - alertInstanceFactory: jest.fn(), - }; - const params = { anomalySeverityType: ANOMALY_SEVERITY.MINOR }; - - await alertExecutor!({ services, params }); - expect(services.callCluster).not.toHaveBeenCalled(); expect(services.alertInstanceFactory).not.toHaveBeenCalled(); }); - }); - describe('sends alert', () => { - it('with service name, environment and transaction type', async () => { + it('anomaly is less than threshold', async () => { jest.spyOn(GetServiceAnomalies, 'getMLJobs').mockReturnValue( - Promise.resolve([ + Promise.resolve(([ { job_id: '1', - custom_settings: { - job_tags: { - environment: 'production', - }, - }, - } as unknown, + custom_settings: { job_tags: { environment: 'development' } }, + }, { job_id: '2', - custom_settings: { - job_tags: { - environment: 'production', - }, - }, - } as unknown, - ] as Job[]) + custom_settings: { job_tags: { environment: 'production' } }, + }, + ] as unknown) as Job[]) ); - let alertExecutor: any; - - const alerting = { - registerType: ({ executor }) => { - alertExecutor = executor; - }, - } as AlertingPlugin['setup']; + const { services, dependencies, executor } = createRuleTypeMocks(); const ml = ({ mlSystemProvider: () => ({ mlAnomalySearch: () => ({ - hits: { total: { value: 2 } }, aggregations: { - services: { + anomaly_groups: { buckets: [ { - key: 'foo', - transaction_types: { - buckets: [{ key: 'type-foo' }], - }, - record_avg: { value: 80 }, - }, - { - key: 'bar', - transaction_types: { - buckets: [{ key: 'type-bar' }], + doc_count: 1, + latest_score: { + top: [{ metrics: { record_score: 0, job_id: '1' } }], }, - record_avg: { value: 20 }, }, ], }, @@ -199,84 +98,77 @@ describe('Transaction duration anomaly alert', () => { } as unknown) as MlPluginSetup; registerTransactionDurationAnomalyAlertType({ - alerting, + ...dependencies, ml, - config$: mockedConfig$, }); - expect(alertExecutor).toBeDefined(); - const scheduleActions = jest.fn(); - const services = { - callCluster: jest.fn(), - alertInstanceFactory: jest.fn(() => ({ scheduleActions })), - }; const params = { anomalySeverityType: ANOMALY_SEVERITY.MINOR }; - await alertExecutor!({ services, params }); - - await alertExecutor!({ services, params }); - [ - 'apm.transaction_duration_anomaly_foo_production_type-foo', - 'apm.transaction_duration_anomaly_bar_production_type-bar', - ].forEach((instanceName) => - expect(services.alertInstanceFactory).toHaveBeenCalledWith(instanceName) - ); + await executor({ params }); - expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { - serviceName: 'foo', - transactionType: 'type-foo', - environment: 'production', - threshold: 'minor', - thresholdValue: 'critical', - }); - expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { - serviceName: 'bar', - transactionType: 'type-bar', - environment: 'production', - threshold: 'minor', - thresholdValue: 'warning', - }); + expect( + services.scopedClusterClient.asCurrentUser.search + ).not.toHaveBeenCalled(); + expect(services.alertInstanceFactory).not.toHaveBeenCalled(); }); + }); - it('with service name', async () => { + describe('sends alert', () => { + it('for all services that exceeded the threshold', async () => { jest.spyOn(GetServiceAnomalies, 'getMLJobs').mockReturnValue( - Promise.resolve([ + Promise.resolve(([ { job_id: '1', - custom_settings: { - job_tags: { - environment: 'production', - }, - }, - } as unknown, + custom_settings: { job_tags: { environment: 'development' } }, + }, { job_id: '2', - custom_settings: { - job_tags: { - environment: 'testing', - }, - }, - } as unknown, - ] as Job[]) + custom_settings: { job_tags: { environment: 'production' } }, + }, + ] as unknown) as Job[]) ); - let alertExecutor: any; - - const alerting = { - registerType: ({ executor }) => { - alertExecutor = executor; - }, - } as AlertingPlugin['setup']; + const { + services, + dependencies, + executor, + scheduleActions, + } = createRuleTypeMocks(); const ml = ({ mlSystemProvider: () => ({ mlAnomalySearch: () => ({ - hits: { total: { value: 2 } }, aggregations: { - services: { + anomaly_groups: { buckets: [ - { key: 'foo', record_avg: { value: 80 } }, - { key: 'bar', record_avg: { value: 20 } }, + { + latest_score: { + top: [ + { + metrics: { + record_score: 80, + job_id: '1', + partition_field_value: 'foo', + by_field_value: 'type-foo', + }, + }, + ], + }, + }, + { + latest_score: { + top: [ + { + metrics: { + record_score: 20, + job_id: '2', + parttition_field_value: 'bar', + by_field_value: 'type-bar', + }, + }, + ], + }, + }, ], }, }, @@ -286,58 +178,26 @@ describe('Transaction duration anomaly alert', () => { } as unknown) as MlPluginSetup; registerTransactionDurationAnomalyAlertType({ - alerting, + ...dependencies, ml, - config$: mockedConfig$, }); - expect(alertExecutor).toBeDefined(); - const scheduleActions = jest.fn(); - const services = { - callCluster: jest.fn(), - alertInstanceFactory: jest.fn(() => ({ scheduleActions })), - }; const params = { anomalySeverityType: ANOMALY_SEVERITY.MINOR }; - await alertExecutor!({ services, params }); + await executor({ params }); + + expect(services.alertInstanceFactory).toHaveBeenCalledTimes(1); - await alertExecutor!({ services, params }); - [ - 'apm.transaction_duration_anomaly_foo_production', - 'apm.transaction_duration_anomaly_foo_testing', - 'apm.transaction_duration_anomaly_bar_production', - 'apm.transaction_duration_anomaly_bar_testing', - ].forEach((instanceName) => - expect(services.alertInstanceFactory).toHaveBeenCalledWith(instanceName) + expect(services.alertInstanceFactory).toHaveBeenCalledWith( + 'apm.transaction_duration_anomaly_foo_development_type-foo' ); expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { serviceName: 'foo', - transactionType: undefined, - environment: 'production', - threshold: 'minor', - thresholdValue: 'critical', - }); - expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { - serviceName: 'bar', - transactionType: undefined, - environment: 'production', - threshold: 'minor', - thresholdValue: 'warning', - }); - expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { - serviceName: 'foo', - transactionType: undefined, - environment: 'testing', - threshold: 'minor', - thresholdValue: 'critical', - }); - expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { - serviceName: 'bar', - transactionType: undefined, - environment: 'testing', + transactionType: 'type-foo', + environment: 'development', threshold: 'minor', - thresholdValue: 'warning', + triggerValue: 'critical', }); }); }); diff --git a/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.ts b/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.ts index 84c3ec7325fd2..15f4a8ea07801 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.ts @@ -6,9 +6,16 @@ */ import { schema } from '@kbn/config-schema'; -import { Observable } from 'rxjs'; -import { isEmpty } from 'lodash'; +import { compact } from 'lodash'; +import { ESSearchResponse } from 'typings/elasticsearch'; +import { QueryContainer } from '@elastic/elasticsearch/api/types'; import { getSeverity } from '../../../common/anomaly_detection'; +import { + SERVICE_ENVIRONMENT, + SERVICE_NAME, + TRANSACTION_TYPE, +} from '../../../common/elasticsearch_fieldnames'; +import { asMutableArray } from '../../../common/utils/as_mutable_array'; import { ANOMALY_SEVERITY } from '../../../../ml/common'; import { KibanaRequest } from '../../../../../../src/core/server'; import { @@ -16,17 +23,11 @@ import { ALERT_TYPES_CONFIG, ANOMALY_ALERT_SEVERITY_TYPES, } from '../../../common/alert_types'; -import { AlertingPlugin } from '../../../../alerting/server'; -import { APMConfig } from '../..'; -import { MlPluginSetup } from '../../../../ml/server'; import { getMLJobs } from '../service_map/get_service_anomalies'; import { apmActionVariables } from './action_variables'; - -interface RegisterAlertParams { - alerting: AlertingPlugin['setup']; - ml?: MlPluginSetup; - config$: Observable; -} +import { RegisterRuleDependencies } from './register_apm_alerts'; +import { parseEnvironmentUrlParam } from '../../../common/environment_filter_values'; +import { createAPMLifecycleRuleType } from './create_apm_lifecycle_rule_type'; const paramsSchema = schema.object({ serviceName: schema.maybe(schema.string()), @@ -46,203 +47,199 @@ const alertTypeConfig = ALERT_TYPES_CONFIG[AlertType.TransactionDurationAnomaly]; export function registerTransactionDurationAnomalyAlertType({ - alerting, + registry, ml, - config$, -}: RegisterAlertParams) { - alerting.registerType({ - id: AlertType.TransactionDurationAnomaly, - name: alertTypeConfig.name, - actionGroups: alertTypeConfig.actionGroups, - defaultActionGroupId: alertTypeConfig.defaultActionGroupId, - validate: { - params: paramsSchema, - }, - actionVariables: { - context: [ - apmActionVariables.serviceName, - apmActionVariables.transactionType, - apmActionVariables.environment, - apmActionVariables.threshold, - apmActionVariables.triggerValue, - ], - }, - producer: 'apm', - minimumLicenseRequired: 'basic', - executor: async ({ services, params, state }) => { - if (!ml) { - return; - } - const alertParams = params; - const request = {} as KibanaRequest; - const { mlAnomalySearch } = ml.mlSystemProvider( - request, - services.savedObjectsClient - ); - const anomalyDetectors = ml.anomalyDetectorsProvider( - request, - services.savedObjectsClient - ); - - const mlJobs = await getMLJobs(anomalyDetectors, alertParams.environment); - - const selectedOption = ANOMALY_ALERT_SEVERITY_TYPES.find( - (option) => option.type === alertParams.anomalySeverityType - ); - - if (!selectedOption) { - throw new Error( - `Anomaly alert severity type ${alertParams.anomalySeverityType} is not supported.` + logger, +}: RegisterRuleDependencies) { + registry.registerType( + createAPMLifecycleRuleType({ + id: AlertType.TransactionDurationAnomaly, + name: alertTypeConfig.name, + actionGroups: alertTypeConfig.actionGroups, + defaultActionGroupId: alertTypeConfig.defaultActionGroupId, + validate: { + params: paramsSchema, + }, + actionVariables: { + context: [ + apmActionVariables.serviceName, + apmActionVariables.transactionType, + apmActionVariables.environment, + apmActionVariables.threshold, + apmActionVariables.triggerValue, + ], + }, + producer: 'apm', + minimumLicenseRequired: 'basic', + executor: async ({ services, params }) => { + if (!ml) { + return {}; + } + const alertParams = params; + const request = {} as KibanaRequest; + const { mlAnomalySearch } = ml.mlSystemProvider( + request, + services.savedObjectsClient + ); + const anomalyDetectors = ml.anomalyDetectorsProvider( + request, + services.savedObjectsClient ); - } - const threshold = selectedOption.threshold; + const mlJobs = await getMLJobs( + anomalyDetectors, + alertParams.environment + ); - if (mlJobs.length === 0) { - return {}; - } - - const jobIds = mlJobs.map((job) => job.job_id); - const anomalySearchParams = { - terminateAfter: 1, - body: { - size: 0, - query: { - bool: { - filter: [ - { term: { result_type: 'record' } }, - { terms: { job_id: jobIds } }, - { - range: { - timestamp: { - gte: `now-${alertParams.windowSize}${alertParams.windowUnit}`, - format: 'epoch_millis', + const selectedOption = ANOMALY_ALERT_SEVERITY_TYPES.find( + (option) => option.type === alertParams.anomalySeverityType + ); + + if (!selectedOption) { + throw new Error( + `Anomaly alert severity type ${alertParams.anomalySeverityType} is not supported.` + ); + } + + const threshold = selectedOption.threshold; + + if (mlJobs.length === 0) { + return {}; + } + + const jobIds = mlJobs.map((job) => job.job_id); + const anomalySearchParams = { + body: { + size: 0, + query: { + bool: { + filter: [ + { term: { result_type: 'record' } }, + { terms: { job_id: jobIds } }, + { term: { is_interim: false } }, + { + range: { + timestamp: { + gte: `now-${alertParams.windowSize}${alertParams.windowUnit}`, + format: 'epoch_millis', + }, }, }, - }, - ...(alertParams.serviceName - ? [ - { - term: { - partition_field_value: alertParams.serviceName, + ...(alertParams.serviceName + ? [ + { + term: { + partition_field_value: alertParams.serviceName, + }, }, - }, - ] - : []), - ...(alertParams.transactionType - ? [ - { - term: { - by_field_value: alertParams.transactionType, + ] + : []), + ...(alertParams.transactionType + ? [ + { + term: { + by_field_value: alertParams.transactionType, + }, }, - }, - ] - : []), - { - range: { - record_score: { - gte: threshold, - }, - }, - }, - ], - }, - }, - aggs: { - services: { - terms: { - field: 'partition_field_value', - size: 50, + ] + : []), + ] as QueryContainer[], }, - aggs: { - transaction_types: { - terms: { - field: 'by_field_value', - }, + }, + aggs: { + anomaly_groups: { + multi_terms: { + terms: [ + { field: 'partition_field_value' }, + { field: 'by_field_value' }, + { field: 'job_id' }, + ], + size: 10000, }, - record_avg: { - avg: { - field: 'record_score', + aggs: { + latest_score: { + top_metrics: { + metrics: asMutableArray([ + { field: 'record_score' }, + { field: 'partition_field_value' }, + { field: 'by_field_value' }, + { field: 'job_id' }, + ] as const), + sort: { + '@timestamp': 'desc' as const, + }, + }, }, }, }, }, }, - }, - }; - - const response = ((await mlAnomalySearch( - anomalySearchParams, - jobIds - )) as unknown) as { - hits: { total: { value: number } }; - aggregations?: { - services: { - buckets: Array<{ - key: string; - record_avg: { value: number }; - transaction_types: { buckets: Array<{ key: string }> }; - }>; - }; }; - }; - - const hitCount = response.hits.total.value; - - if (hitCount > 0) { - function scheduleAction({ - serviceName, - severity, - environment, - transactionType, - }: { - serviceName: string; - severity: string; - environment?: string; - transactionType?: string; - }) { - const alertInstanceName = [ - AlertType.TransactionDurationAnomaly, - serviceName, - environment, - transactionType, - ] - .filter((name) => name) - .join('_'); - - const alertInstance = services.alertInstanceFactory( - alertInstanceName - ); - alertInstance.scheduleActions(alertTypeConfig.defaultActionGroupId, { - serviceName, - environment, - transactionType, - threshold: selectedOption?.label, - thresholdValue: severity, - }); - } - mlJobs.map((job) => { - const environment = job.custom_settings?.job_tags?.environment; - response.aggregations?.services.buckets.forEach((serviceBucket) => { - const serviceName = serviceBucket.key as string; - const severity = getSeverity(serviceBucket.record_avg.value); - if (isEmpty(serviceBucket.transaction_types?.buckets)) { - scheduleAction({ serviceName, severity, environment }); - } else { - serviceBucket.transaction_types?.buckets.forEach((typeBucket) => { - const transactionType = typeBucket.key as string; - scheduleAction({ - serviceName, - severity, - environment, - transactionType, - }); - }); - } - }); + const response: ESSearchResponse< + unknown, + typeof anomalySearchParams + > = (await mlAnomalySearch(anomalySearchParams, [])) as any; + + const anomalies = + response.aggregations?.anomaly_groups.buckets + .map((bucket) => { + const latest = bucket.latest_score.top[0].metrics; + + const job = mlJobs.find((j) => j.job_id === latest.job_id); + + if (!job) { + logger.warn( + `Could not find matching job for job id ${latest.job_id}` + ); + return undefined; + } + + return { + serviceName: latest.partition_field_value as string, + transactionType: latest.by_field_value as string, + environment: job.custom_settings!.job_tags!.environment, + score: latest.record_score as number, + }; + }) + .filter((anomaly) => + anomaly ? anomaly.score >= threshold : false + ) ?? []; + + compact(anomalies).forEach((anomaly) => { + const { serviceName, environment, transactionType, score } = anomaly; + + const parsedEnvironment = parseEnvironmentUrlParam(environment); + + services + .alertWithLifecycle({ + id: [ + AlertType.TransactionDurationAnomaly, + serviceName, + environment, + transactionType, + ] + .filter((name) => name) + .join('_'), + fields: { + [SERVICE_NAME]: serviceName, + ...(parsedEnvironment.esFieldValue + ? { [SERVICE_ENVIRONMENT]: environment } + : {}), + [TRANSACTION_TYPE]: transactionType, + }, + }) + .scheduleActions(alertTypeConfig.defaultActionGroupId, { + serviceName, + transactionType, + environment, + threshold: selectedOption?.label, + triggerValue: getSeverity(score), + }); }); - } - }, - }); + + return {}; + }, + }) + ); } diff --git a/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.test.ts b/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.test.ts index 148cd813a8a22..be5f4705482d0 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.test.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.test.ts @@ -5,48 +5,19 @@ * 2.0. */ -import { Observable } from 'rxjs'; -import * as Rx from 'rxjs'; -import { toArray, map } from 'rxjs/operators'; -import { AlertingPlugin } from '../../../../alerting/server'; -import { APMConfig } from '../..'; import { registerTransactionErrorRateAlertType } from './register_transaction_error_rate_alert_type'; -import { elasticsearchServiceMock } from 'src/core/server/mocks'; // eslint-disable-next-line @kbn/eslint/no-restricted-paths import { elasticsearchClientMock } from 'src/core/server/elasticsearch/client/mocks'; - -type Operator = (source: Rx.Observable) => Rx.Observable; -const pipeClosure = (fn: Operator): Operator => { - return (source: Rx.Observable) => { - return Rx.defer(() => fn(source)); - }; -}; -const mockedConfig$ = (Rx.of('apm_oss.errorIndices').pipe( - pipeClosure((source$) => { - return source$.pipe(map((i) => i)); - }), - toArray() -) as unknown) as Observable; +import { createRuleTypeMocks } from './test_utils'; describe('Transaction error rate alert', () => { it("doesn't send an alert when rate is less than threshold", async () => { - let alertExecutor: any; - const alerting = { - registerType: ({ executor }) => { - alertExecutor = executor; - }, - } as AlertingPlugin['setup']; + const { services, dependencies, executor } = createRuleTypeMocks(); registerTransactionErrorRateAlertType({ - alerting, - config$: mockedConfig$, + ...dependencies, }); - expect(alertExecutor).toBeDefined(); - const services = { - scopedClusterClient: elasticsearchServiceMock.createScopedClusterClient(), - alertInstanceFactory: jest.fn(), - }; const params = { threshold: 1 }; services.scopedClusterClient.asCurrentUser.search.mockReturnValue( @@ -60,6 +31,11 @@ describe('Transaction error rate alert', () => { }, took: 0, timed_out: false, + aggregations: { + series: { + buckets: [], + }, + }, _shards: { failed: 0, skipped: 0, @@ -69,30 +45,21 @@ describe('Transaction error rate alert', () => { }) ); - await alertExecutor!({ services, params }); + await executor({ params }); expect(services.alertInstanceFactory).not.toBeCalled(); }); - it('sends alerts with service name, transaction type and environment', async () => { - let alertExecutor: any; - const alerting = { - registerType: ({ executor }) => { - alertExecutor = executor; - }, - } as AlertingPlugin['setup']; + it('sends alerts for services that exceeded the threshold', async () => { + const { + services, + dependencies, + executor, + scheduleActions, + } = createRuleTypeMocks(); registerTransactionErrorRateAlertType({ - alerting, - config$: mockedConfig$, + ...dependencies, }); - expect(alertExecutor).toBeDefined(); - - const scheduleActions = jest.fn(); - const services = { - scopedClusterClient: elasticsearchServiceMock.createScopedClusterClient(), - alertInstanceFactory: jest.fn(() => ({ scheduleActions })), - }; - const params = { threshold: 10, windowSize: 5, windowUnit: 'm' }; services.scopedClusterClient.asCurrentUser.search.mockReturnValue( elasticsearchClientMock.createSuccessTransportRequestPromise({ @@ -100,37 +67,38 @@ describe('Transaction error rate alert', () => { hits: [], total: { relation: 'eq', - value: 4, + value: 0, }, }, aggregations: { - failed_transactions: { - doc_count: 2, - }, - services: { + series: { buckets: [ { - key: 'foo', - transaction_types: { + key: ['foo', 'env-foo', 'type-foo'], + outcomes: { buckets: [ { - key: 'type-foo', - environments: { - buckets: [{ key: 'env-foo' }, { key: 'env-foo-2' }], - }, + key: 'success', + doc_count: 90, + }, + { + key: 'failure', + doc_count: 10, }, ], }, }, { - key: 'bar', - transaction_types: { + key: ['bar', 'env-bar', 'type-bar'], + outcomes: { buckets: [ { - key: 'type-bar', - environments: { - buckets: [{ key: 'env-bar' }, { key: 'env-bar-2' }], - }, + key: 'success', + doc_count: 90, + }, + { + key: 'failure', + doc_count: 1, }, ], }, @@ -149,208 +117,25 @@ describe('Transaction error rate alert', () => { }) ); - await alertExecutor!({ services, params }); - [ - 'apm.transaction_error_rate_foo_type-foo_env-foo', - 'apm.transaction_error_rate_foo_type-foo_env-foo-2', - 'apm.transaction_error_rate_bar_type-bar_env-bar', - 'apm.transaction_error_rate_bar_type-bar_env-bar-2', - ].forEach((instanceName) => - expect(services.alertInstanceFactory).toHaveBeenCalledWith(instanceName) - ); - - expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { - serviceName: 'foo', - transactionType: 'type-foo', - environment: 'env-foo', - threshold: 10, - triggerValue: '50', - interval: '5m', - }); - expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { - serviceName: 'foo', - transactionType: 'type-foo', - environment: 'env-foo-2', - threshold: 10, - triggerValue: '50', - interval: '5m', - }); - expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { - serviceName: 'bar', - transactionType: 'type-bar', - environment: 'env-bar', - threshold: 10, - triggerValue: '50', - interval: '5m', - }); - expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { - serviceName: 'bar', - transactionType: 'type-bar', - environment: 'env-bar-2', - threshold: 10, - triggerValue: '50', - interval: '5m', - }); - }); - it('sends alerts with service name and transaction type', async () => { - let alertExecutor: any; - const alerting = { - registerType: ({ executor }) => { - alertExecutor = executor; - }, - } as AlertingPlugin['setup']; - - registerTransactionErrorRateAlertType({ - alerting, - config$: mockedConfig$, - }); - expect(alertExecutor).toBeDefined(); - - const scheduleActions = jest.fn(); - const services = { - scopedClusterClient: elasticsearchServiceMock.createScopedClusterClient(), - alertInstanceFactory: jest.fn(() => ({ scheduleActions })), - }; const params = { threshold: 10, windowSize: 5, windowUnit: 'm' }; - services.scopedClusterClient.asCurrentUser.search.mockReturnValue( - elasticsearchClientMock.createSuccessTransportRequestPromise({ - hits: { - hits: [], - total: { - relation: 'eq', - value: 4, - }, - }, - aggregations: { - failed_transactions: { - doc_count: 2, - }, - services: { - buckets: [ - { - key: 'foo', - transaction_types: { - buckets: [{ key: 'type-foo' }], - }, - }, - { - key: 'bar', - transaction_types: { - buckets: [{ key: 'type-bar' }], - }, - }, - ], - }, - }, - took: 0, - timed_out: false, - _shards: { - failed: 0, - skipped: 0, - successful: 1, - total: 1, - }, - }) - ); - - await alertExecutor!({ services, params }); - [ - 'apm.transaction_error_rate_foo_type-foo', - 'apm.transaction_error_rate_bar_type-bar', - ].forEach((instanceName) => - expect(services.alertInstanceFactory).toHaveBeenCalledWith(instanceName) - ); - - expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { - serviceName: 'foo', - transactionType: 'type-foo', - environment: undefined, - threshold: 10, - triggerValue: '50', - interval: '5m', - }); - expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { - serviceName: 'bar', - transactionType: 'type-bar', - environment: undefined, - threshold: 10, - triggerValue: '50', - interval: '5m', - }); - }); - - it('sends alerts with service name', async () => { - let alertExecutor: any; - const alerting = { - registerType: ({ executor }) => { - alertExecutor = executor; - }, - } as AlertingPlugin['setup']; - - registerTransactionErrorRateAlertType({ - alerting, - config$: mockedConfig$, - }); - expect(alertExecutor).toBeDefined(); + await executor({ params }); - const scheduleActions = jest.fn(); - const services = { - scopedClusterClient: elasticsearchServiceMock.createScopedClusterClient(), - alertInstanceFactory: jest.fn(() => ({ scheduleActions })), - }; - const params = { threshold: 10, windowSize: 5, windowUnit: 'm' }; + expect(services.alertInstanceFactory).toHaveBeenCalledTimes(1); - services.scopedClusterClient.asCurrentUser.search.mockReturnValue( - elasticsearchClientMock.createSuccessTransportRequestPromise({ - hits: { - hits: [], - total: { - value: 4, - relation: 'eq', - }, - }, - aggregations: { - failed_transactions: { - doc_count: 2, - }, - services: { - buckets: [{ key: 'foo' }, { key: 'bar' }], - }, - }, - took: 0, - timed_out: false, - _shards: { - failed: 0, - skipped: 0, - successful: 1, - total: 1, - }, - }) + expect(services.alertInstanceFactory).toHaveBeenCalledWith( + 'apm.transaction_error_rate_foo_type-foo_env-foo' ); - - await alertExecutor!({ services, params }); - [ - 'apm.transaction_error_rate_foo', - 'apm.transaction_error_rate_bar', - ].forEach((instanceName) => - expect(services.alertInstanceFactory).toHaveBeenCalledWith(instanceName) + expect(services.alertInstanceFactory).not.toHaveBeenCalledWith( + 'apm.transaction_error_rate_bar_type-bar_env-bar' ); expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { serviceName: 'foo', - transactionType: undefined, - environment: undefined, - threshold: 10, - triggerValue: '50', - interval: '5m', - }); - expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { - serviceName: 'bar', - transactionType: undefined, - environment: undefined, + transactionType: 'type-foo', + environment: 'env-foo', threshold: 10, - triggerValue: '50', + triggerValue: '10', interval: '5m', }); }); diff --git a/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.ts b/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.ts index 0b2684cdaf083..0865bed41142e 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.ts @@ -6,11 +6,7 @@ */ import { schema } from '@kbn/config-schema'; -import { isEmpty } from 'lodash'; -import { Observable } from 'rxjs'; import { take } from 'rxjs/operators'; -import { APMConfig } from '../..'; -import { AlertingPlugin } from '../../../../alerting/server'; import { AlertType, ALERT_TYPES_CONFIG } from '../../../common/alert_types'; import { EVENT_OUTCOME, @@ -26,11 +22,8 @@ import { environmentQuery } from '../../../server/utils/queries'; import { getApmIndices } from '../settings/apm_indices/get_apm_indices'; import { apmActionVariables } from './action_variables'; import { alertingEsClient } from './alerting_es_client'; - -interface RegisterAlertParams { - alerting: AlertingPlugin['setup']; - config$: Observable; -} +import { createAPMLifecycleRuleType } from './create_apm_lifecycle_rule_type'; +import { RegisterRuleDependencies } from './register_apm_alerts'; const paramsSchema = schema.object({ windowSize: schema.number(), @@ -44,158 +37,165 @@ const paramsSchema = schema.object({ const alertTypeConfig = ALERT_TYPES_CONFIG[AlertType.TransactionErrorRate]; export function registerTransactionErrorRateAlertType({ - alerting, + registry, config$, -}: RegisterAlertParams) { - alerting.registerType({ - id: AlertType.TransactionErrorRate, - name: alertTypeConfig.name, - actionGroups: alertTypeConfig.actionGroups, - defaultActionGroupId: alertTypeConfig.defaultActionGroupId, - validate: { - params: paramsSchema, - }, - actionVariables: { - context: [ - apmActionVariables.transactionType, - apmActionVariables.serviceName, - apmActionVariables.environment, - apmActionVariables.threshold, - apmActionVariables.triggerValue, - apmActionVariables.interval, - ], - }, - producer: 'apm', - minimumLicenseRequired: 'basic', - executor: async ({ services, params: alertParams }) => { - const config = await config$.pipe(take(1)).toPromise(); - const indices = await getApmIndices({ - config, - savedObjectsClient: services.savedObjectsClient, - }); - const maxServiceEnvironments = config['xpack.apm.maxServiceEnvironments']; +}: RegisterRuleDependencies) { + registry.registerType( + createAPMLifecycleRuleType({ + id: AlertType.TransactionErrorRate, + name: alertTypeConfig.name, + actionGroups: alertTypeConfig.actionGroups, + defaultActionGroupId: alertTypeConfig.defaultActionGroupId, + validate: { + params: paramsSchema, + }, + actionVariables: { + context: [ + apmActionVariables.transactionType, + apmActionVariables.serviceName, + apmActionVariables.environment, + apmActionVariables.threshold, + apmActionVariables.triggerValue, + apmActionVariables.interval, + ], + }, + producer: 'apm', + minimumLicenseRequired: 'basic', + executor: async ({ services, params: alertParams }) => { + const config = await config$.pipe(take(1)).toPromise(); + const indices = await getApmIndices({ + config, + savedObjectsClient: services.savedObjectsClient, + }); - const searchParams = { - index: indices['apm_oss.transactionIndices'], - size: 0, - body: { - track_total_hits: true, - query: { - bool: { - filter: [ - { - range: { - '@timestamp': { - gte: `now-${alertParams.windowSize}${alertParams.windowUnit}`, + const searchParams = { + index: indices['apm_oss.transactionIndices'], + size: 1, + body: { + query: { + bool: { + filter: [ + { + range: { + '@timestamp': { + gte: `now-${alertParams.windowSize}${alertParams.windowUnit}`, + }, }, }, - }, - { term: { [PROCESSOR_EVENT]: ProcessorEvent.transaction } }, - ...(alertParams.serviceName - ? [{ term: { [SERVICE_NAME]: alertParams.serviceName } }] - : []), - ...(alertParams.transactionType - ? [ - { - term: { - [TRANSACTION_TYPE]: alertParams.transactionType, + { term: { [PROCESSOR_EVENT]: ProcessorEvent.transaction } }, + { + terms: { + [EVENT_OUTCOME]: [ + EventOutcome.failure, + EventOutcome.success, + ], + }, + }, + ...(alertParams.serviceName + ? [{ term: { [SERVICE_NAME]: alertParams.serviceName } }] + : []), + ...(alertParams.transactionType + ? [ + { + term: { + [TRANSACTION_TYPE]: alertParams.transactionType, + }, }, - }, - ] - : []), - ...environmentQuery(alertParams.environment), - ], - }, - }, - aggs: { - failed_transactions: { - filter: { term: { [EVENT_OUTCOME]: EventOutcome.failure } }, - }, - services: { - terms: { - field: SERVICE_NAME, - size: 50, + ] + : []), + ...environmentQuery(alertParams.environment), + ], }, - aggs: { - transaction_types: { - terms: { field: TRANSACTION_TYPE }, - aggs: { - environments: { - terms: { - field: SERVICE_ENVIRONMENT, - size: maxServiceEnvironments, - }, + }, + aggs: { + series: { + multi_terms: { + terms: [ + { field: SERVICE_NAME }, + { field: SERVICE_ENVIRONMENT, missing: '' }, + { field: TRANSACTION_TYPE }, + ], + size: 10000, + }, + aggs: { + outcomes: { + terms: { + field: EVENT_OUTCOME, }, }, }, }, }, }, - }, - }; + }; - const { body: response } = await alertingEsClient(services, searchParams); - if (!response.aggregations) { - return; - } + const response = await alertingEsClient( + services.scopedClusterClient, + searchParams + ); - const failedTransactionCount = - response.aggregations.failed_transactions.doc_count; - const totalTransactionCount = response.hits.total.value; - const transactionErrorRate = - (failedTransactionCount / totalTransactionCount) * 100; + if (!response.aggregations) { + return {}; + } - if (transactionErrorRate > alertParams.threshold) { - function scheduleAction({ - serviceName, - environment, - transactionType, - }: { - serviceName: string; - environment?: string; - transactionType?: string; - }) { - const alertInstanceName = [ - AlertType.TransactionErrorRate, - serviceName, - transactionType, - environment, - ] - .filter((name) => name) - .join('_'); + const results = response.aggregations.series.buckets + .map((bucket) => { + const [serviceName, environment, transactionType] = bucket.key; + + const failed = + bucket.outcomes.buckets.find( + (outcomeBucket) => outcomeBucket.key === EventOutcome.failure + )?.doc_count ?? 0; + const succesful = + bucket.outcomes.buckets.find( + (outcomeBucket) => outcomeBucket.key === EventOutcome.success + )?.doc_count ?? 0; - const alertInstance = services.alertInstanceFactory( - alertInstanceName - ); - alertInstance.scheduleActions(alertTypeConfig.defaultActionGroupId, { + return { + serviceName, + environment, + transactionType, + errorRate: (failed / (failed + succesful)) * 100, + }; + }) + .filter((result) => result.errorRate >= alertParams.threshold); + + results.forEach((result) => { + const { serviceName, - transactionType, environment, - threshold: alertParams.threshold, - triggerValue: asDecimalOrInteger(transactionErrorRate), - interval: `${alertParams.windowSize}${alertParams.windowUnit}`, - }); - } + transactionType, + errorRate, + } = result; - response.aggregations?.services.buckets.forEach((serviceBucket) => { - const serviceName = serviceBucket.key as string; - if (isEmpty(serviceBucket.transaction_types?.buckets)) { - scheduleAction({ serviceName }); - } else { - serviceBucket.transaction_types.buckets.forEach((typeBucket) => { - const transactionType = typeBucket.key as string; - if (isEmpty(typeBucket.environments?.buckets)) { - scheduleAction({ serviceName, transactionType }); - } else { - typeBucket.environments.buckets.forEach((envBucket) => { - const environment = envBucket.key as string; - scheduleAction({ serviceName, transactionType, environment }); - }); - } + services + .alertWithLifecycle({ + id: [ + AlertType.TransactionErrorRate, + serviceName, + transactionType, + environment, + ] + .filter((name) => name) + .join('_'), + fields: { + [SERVICE_NAME]: serviceName, + ...(environment ? { [SERVICE_ENVIRONMENT]: environment } : {}), + [TRANSACTION_TYPE]: transactionType, + }, + }) + .scheduleActions(alertTypeConfig.defaultActionGroupId, { + serviceName, + transactionType, + environment, + threshold: alertParams.threshold, + triggerValue: asDecimalOrInteger(errorRate), + interval: `${alertParams.windowSize}${alertParams.windowUnit}`, }); - } }); - } - }, - }); + + return {}; + }, + }) + ); } diff --git a/x-pack/plugins/apm/server/lib/alerts/test_utils/index.ts b/x-pack/plugins/apm/server/lib/alerts/test_utils/index.ts new file mode 100644 index 0000000000000..37b3e282d0a59 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/alerts/test_utils/index.ts @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Logger } from 'kibana/server'; +import { of } from 'rxjs'; +import { elasticsearchServiceMock } from 'src/core/server/mocks'; +import { APMConfig } from '../../..'; +import { APMRuleRegistry } from '../../../plugin'; + +export const createRuleTypeMocks = () => { + let alertExecutor: (...args: any[]) => Promise; + + const mockedConfig$ = of({ + /* eslint-disable @typescript-eslint/naming-convention */ + 'apm_oss.errorIndices': 'apm-*', + 'apm_oss.transactionIndices': 'apm-*', + /* eslint-enable @typescript-eslint/naming-convention */ + } as APMConfig); + + const loggerMock = ({ + debug: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + } as unknown) as Logger; + + const registry = { + registerType: ({ executor }) => { + alertExecutor = executor; + }, + } as APMRuleRegistry; + + const scheduleActions = jest.fn(); + + const services = { + scopedClusterClient: elasticsearchServiceMock.createScopedClusterClient(), + scopedRuleRegistryClient: { + bulkIndex: jest.fn(), + }, + alertInstanceFactory: jest.fn(() => ({ scheduleActions })), + alertWithLifecycle: jest.fn(), + logger: loggerMock, + }; + + return { + dependencies: { + registry, + config$: mockedConfig$, + logger: loggerMock, + }, + services, + scheduleActions, + executor: async ({ params }: { params: Record }) => { + return alertExecutor({ + services, + params, + startedAt: new Date(), + }); + }, + }; +}; diff --git a/x-pack/plugins/apm/server/plugin.ts b/x-pack/plugins/apm/server/plugin.ts index 074df7eaafd3c..cb94b18a1ecf9 100644 --- a/x-pack/plugins/apm/server/plugin.ts +++ b/x-pack/plugins/apm/server/plugin.ts @@ -43,6 +43,8 @@ import { import { registerRoutes } from './routes/register_routes'; import { getGlobalApmServerRouteRepository } from './routes/get_global_apm_server_route_repository'; +export type APMRuleRegistry = ReturnType['ruleRegistry']; + export class APMPlugin implements Plugin< @@ -72,15 +74,6 @@ export class APMPlugin core.uiSettings.register(uiSettings); - if (plugins.actions && plugins.alerting) { - registerApmAlerts({ - alerting: plugins.alerting, - actions: plugins.actions, - ml: plugins.ml, - config$: mergedConfig$, - }); - } - const currentConfig = mergeConfigs( plugins.apmOss.config, this.initContext.config.get() @@ -157,6 +150,28 @@ export class APMPlugin config: await mergedConfig$.pipe(take(1)).toPromise(), }); + const apmRuleRegistry = plugins.observability.ruleRegistry.create({ + name: 'apm', + fieldMap: { + 'service.environment': { + type: 'keyword', + }, + 'transaction.type': { + type: 'keyword', + }, + 'processor.event': { + type: 'keyword', + }, + }, + }); + + registerApmAlerts({ + registry: apmRuleRegistry, + ml: plugins.ml, + config$: mergedConfig$, + logger: this.logger!.get('rule'), + }); + return { config$: mergedConfig$, getApmIndices: boundGetApmIndices, @@ -186,6 +201,7 @@ export class APMPlugin }, }); }, + ruleRegistry: apmRuleRegistry, }; } diff --git a/x-pack/plugins/apm/server/routes/typings.ts b/x-pack/plugins/apm/server/routes/typings.ts index 0fec88a4326c3..517387c5f74ef 100644 --- a/x-pack/plugins/apm/server/routes/typings.ts +++ b/x-pack/plugins/apm/server/routes/typings.ts @@ -60,21 +60,3 @@ export interface APMRouteHandlerResources { }; }; } - -// export type Client< -// TRouteState, -// TOptions extends { abortable: boolean } = { abortable: true } -// > = ( -// options: Omit< -// FetchOptions, -// 'query' | 'body' | 'pathname' | 'method' | 'signal' -// > & { -// forceCache?: boolean; -// endpoint: TEndpoint; -// } & MaybeParams & -// (TOptions extends { abortable: true } ? { signal: AbortSignal | null } : {}) -// ) => Promise< -// TRouteState[TEndpoint] extends { ret: any } -// ? TRouteState[TEndpoint]['ret'] -// : unknown -// >; diff --git a/x-pack/plugins/apm/server/types.ts b/x-pack/plugins/apm/server/types.ts index cef9eaf2f4fc0..dbc220f9f6b15 100644 --- a/x-pack/plugins/apm/server/types.ts +++ b/x-pack/plugins/apm/server/types.ts @@ -125,6 +125,7 @@ const requiredDependencies = [ 'triggersActionsUi', 'embeddable', 'infra', + 'observability', ] as const; const optionalDependencies = [ @@ -134,7 +135,6 @@ const optionalDependencies = [ 'taskManager', 'actions', 'alerting', - 'observability', 'security', 'ml', 'home', diff --git a/x-pack/plugins/apm/tsconfig.json b/x-pack/plugins/apm/tsconfig.json index ffbf11c23f63a..bb341059e2d43 100644 --- a/x-pack/plugins/apm/tsconfig.json +++ b/x-pack/plugins/apm/tsconfig.json @@ -38,6 +38,7 @@ { "path": "../ml/tsconfig.json" }, { "path": "../observability/tsconfig.json" }, { "path": "../reporting/tsconfig.json" }, + { "path": "../rule_registry/tsconfig.json" }, { "path": "../security/tsconfig.json" }, { "path": "../task_manager/tsconfig.json" }, { "path": "../triggers_actions_ui/tsconfig.json" } diff --git a/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts b/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts index efe3186f97805..cd87bc8e6e18b 100644 --- a/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts +++ b/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts @@ -12,7 +12,6 @@ import { IClusterClientAdapter, EVENT_BUFFER_LENGTH, } from './cluster_client_adapter'; -import { contextMock } from './context.mock'; import { findOptionsSchema } from '../event_log_client'; import { delay } from '../lib/delay'; import { times } from 'lodash'; @@ -31,7 +30,7 @@ beforeEach(() => { clusterClientAdapter = new ClusterClientAdapter({ logger, elasticsearchClientPromise: Promise.resolve(clusterClient), - context: contextMock.create(), + wait: () => Promise.resolve(true), }); }); diff --git a/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts b/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts index 5d7be2278d55d..dd6ac6350d6e3 100644 --- a/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts +++ b/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts @@ -10,8 +10,8 @@ import { bufferTime, filter as rxFilter, switchMap } from 'rxjs/operators'; import { reject, isUndefined, isNumber } from 'lodash'; import type { PublicMethodsOf } from '@kbn/utility-types'; import { Logger, ElasticsearchClient } from 'src/core/server'; +import util from 'util'; import { estypes } from '@elastic/elasticsearch'; -import { EsContext } from '.'; import { IEvent, IValidatedEvent, SAVED_OBJECT_REL_PRIMARY } from '../types'; import { FindOptionsType } from '../event_log_client'; import { esKuery } from '../../../../../src/plugins/data/server'; @@ -26,10 +26,12 @@ export interface Doc { body: IEvent; } +type Wait = () => Promise; + export interface ConstructorOpts { logger: Logger; elasticsearchClientPromise: Promise; - context: EsContext; + wait: Wait; } export interface QueryEventsBySavedObjectResult { @@ -39,18 +41,21 @@ export interface QueryEventsBySavedObjectResult { data: IValidatedEvent[]; } -export class ClusterClientAdapter { +// eslint-disable-next-line @typescript-eslint/no-explicit-any +type AliasAny = any; + +export class ClusterClientAdapter { private readonly logger: Logger; private readonly elasticsearchClientPromise: Promise; - private readonly docBuffer$: Subject; - private readonly context: EsContext; + private readonly docBuffer$: Subject; + private readonly wait: Wait; private readonly docsBufferedFlushed: Promise; constructor(opts: ConstructorOpts) { this.logger = opts.logger; this.elasticsearchClientPromise = opts.elasticsearchClientPromise; - this.context = opts.context; - this.docBuffer$ = new Subject(); + this.wait = opts.wait; + this.docBuffer$ = new Subject(); // buffer event log docs for time / buffer length, ignore empty // buffers, then index the buffered docs; kick things off with a @@ -75,18 +80,21 @@ export class ClusterClientAdapter { await this.docsBufferedFlushed; } - public indexDocument(doc: Doc): void { + public indexDocument(doc: TDoc): void { this.docBuffer$.next(doc); } - async indexDocuments(docs: Doc[]): Promise { + async indexDocuments(docs: TDoc[]): Promise { // If es initialization failed, don't try to index. // Also, don't log here, we log the failure case in plugin startup // instead, otherwise we'd be spamming the log (if done here) - if (!(await this.context.waitTillReady())) { + if (!(await this.wait())) { + this.logger.debug(`Initialization failed, not indexing ${docs.length} documents`); return; } + this.logger.debug(`Indexing ${docs.length} documents`); + const bulkBody: Array> = []; for (const doc of docs) { @@ -98,7 +106,13 @@ export class ClusterClientAdapter { try { const esClient = await this.elasticsearchClientPromise; - await esClient.bulk({ body: bulkBody }); + const response = await esClient.bulk({ body: bulkBody }); + + if (response.body.errors) { + const error = new Error('Error writing some bulk events'); + error.stack += '\n' + util.inspect(response.body.items, { depth: null }); + this.logger.error(error); + } } catch (err) { this.logger.error( `error writing bulk events: "${err.message}"; docs: ${JSON.stringify(bulkBody)}` @@ -156,7 +170,9 @@ export class ClusterClientAdapter { // instances at the same time. const existsNow = await this.doesIndexTemplateExist(name); if (!existsNow) { - throw new Error(`error creating index template: ${err.message}`); + const error = new Error(`error creating index template: ${err.message}`); + Object.assign(error, { wrapped: err }); + throw error; } } } diff --git a/x-pack/plugins/event_log/server/es/context.ts b/x-pack/plugins/event_log/server/es/context.ts index 6d3b2208b3408..f6ae0a2002dd4 100644 --- a/x-pack/plugins/event_log/server/es/context.ts +++ b/x-pack/plugins/event_log/server/es/context.ts @@ -53,7 +53,7 @@ class EsContextImpl implements EsContext { this.esAdapter = new ClusterClientAdapter({ logger: params.logger, elasticsearchClientPromise: params.elasticsearchClientPromise, - context: this, + wait: () => this.readySignal.wait(), }); } diff --git a/x-pack/plugins/event_log/server/index.ts b/x-pack/plugins/event_log/server/index.ts index ab2b9709703c5..4c5513a7fc59c 100644 --- a/x-pack/plugins/event_log/server/index.ts +++ b/x-pack/plugins/event_log/server/index.ts @@ -20,5 +20,9 @@ export { SAVED_OBJECT_REL_PRIMARY, } from './types'; +export { ClusterClientAdapter } from './es/cluster_client_adapter'; + +export { createReadySignal } from './lib/ready_signal'; + export const config = { schema: ConfigSchema }; export const plugin = (context: PluginInitializerContext) => new Plugin(context); diff --git a/x-pack/plugins/observability/kibana.json b/x-pack/plugins/observability/kibana.json index 5c47d0376581a..74efc1f4985a3 100644 --- a/x-pack/plugins/observability/kibana.json +++ b/x-pack/plugins/observability/kibana.json @@ -3,7 +3,7 @@ "version": "8.0.0", "kibanaVersion": "kibana", "configPath": ["xpack", "observability"], - "optionalPlugins": ["licensing", "home", "usageCollection","lens"], + "optionalPlugins": ["licensing", "home", "usageCollection","lens", "ruleRegistry"], "requiredPlugins": ["data"], "ui": true, "server": true, diff --git a/x-pack/plugins/observability/server/plugin.ts b/x-pack/plugins/observability/server/plugin.ts index cb9878f578885..c59b4dbe373dd 100644 --- a/x-pack/plugins/observability/server/plugin.ts +++ b/x-pack/plugins/observability/server/plugin.ts @@ -6,29 +6,30 @@ */ import { PluginInitializerContext, Plugin, CoreSetup } from 'src/core/server'; +import { pickWithPatterns } from '../../rule_registry/server'; import { ObservabilityConfig } from '.'; import { bootstrapAnnotations, - ScopedAnnotationsClient, ScopedAnnotationsClientFactory, AnnotationsAPI, } from './lib/annotations/bootstrap_annotations'; +import type { RuleRegistryPluginSetupContract } from '../../rule_registry/server'; import { uiSettings } from './ui_settings'; +import { ecsFieldMap } from '../../rule_registry/server'; -type LazyScopedAnnotationsClientFactory = ( - ...args: Parameters -) => Promise; - -export interface ObservabilityPluginSetup { - getScopedAnnotationsClient: LazyScopedAnnotationsClientFactory; -} +export type ObservabilityPluginSetup = ReturnType; export class ObservabilityPlugin implements Plugin { constructor(private readonly initContext: PluginInitializerContext) { this.initContext = initContext; } - public setup(core: CoreSetup, plugins: {}): ObservabilityPluginSetup { + public setup( + core: CoreSetup, + plugins: { + ruleRegistry: RuleRegistryPluginSetupContract; + } + ) { const config = this.initContext.config.get(); let annotationsApiPromise: Promise | undefined; @@ -48,10 +49,16 @@ export class ObservabilityPlugin implements Plugin { } return { - getScopedAnnotationsClient: async (...args) => { + getScopedAnnotationsClient: async (...args: Parameters) => { const api = await annotationsApiPromise; return api?.getScopedAnnotationsClient(...args); }, + ruleRegistry: plugins.ruleRegistry.create({ + name: 'observability', + fieldMap: { + ...pickWithPatterns(ecsFieldMap, 'host.name', 'service.name'), + }, + }), }; } diff --git a/x-pack/plugins/observability/tsconfig.json b/x-pack/plugins/observability/tsconfig.json index f55ae640a8026..bd37bc09bc130 100644 --- a/x-pack/plugins/observability/tsconfig.json +++ b/x-pack/plugins/observability/tsconfig.json @@ -23,6 +23,7 @@ { "path": "../../../src/plugins/kibana_utils/tsconfig.json" }, { "path": "../../../src/plugins/usage_collection/tsconfig.json" }, { "path": "../alerting/tsconfig.json" }, + { "path": "../rule_registry/tsconfig.json" }, { "path": "../licensing/tsconfig.json" }, { "path": "../lens/tsconfig.json" }, { "path": "../translations/tsconfig.json" } diff --git a/x-pack/plugins/rule_registry/README.md b/x-pack/plugins/rule_registry/README.md new file mode 100644 index 0000000000000..17fe2b20f74fa --- /dev/null +++ b/x-pack/plugins/rule_registry/README.md @@ -0,0 +1,68 @@ +The rule registry plugin aims to make it easy for rule type producers to have their rules produce the data that they need to build rich experiences on top of a unified experience, without the risk of mapping conflicts. + +A rule registry creates a template, an ILM policy, and an alias. The template mappings can be configured. It also injects a client scoped to these indices. + +It also supports inheritance, which means that producers can create a registry specific to their solution or rule type, and specify additional mappings to be used. + +The rule registry plugin creates a root rule registry, with the mappings defined needed to create a unified experience. Rule type producers can use the plugin to access the root rule registry, and create their own registry that branches off of the root rule registry. The rule registry client sees data from its own registry, and all registries that branches off of it. It does not see data from its parents. + +Creating a rule registry + +To create a rule registry, producers should add the `ruleRegistry` plugin to their dependencies. They can then use the `ruleRegistry.create` method to create a child registry, with the additional mappings that should be used by specifying `fieldMap`: + +```ts +const observabilityRegistry = plugins.ruleRegistry.create({ + name: 'observability', + fieldMap: { + ...pickWithPatterns(ecsFieldMap, 'host.name', 'service.name'), + }, +}) +``` + +`fieldMap` is a key-value map of field names and mapping options: + +```ts +{ + '@timestamp': { + type: 'date', + array: false, + required: true, + } +} +``` + +ECS mappings are generated via a script in the rule registry plugin directory. These mappings are available in x-pack/plugins/rule_registry/server/generated/ecs_field_map.ts. + +To pick many fields, you can use `pickWithPatterns`, which supports wildcards with full type support. + +If a registry is created, it will initialise as soon as the core services needed become available. It will create a (versioned) template, alias, and ILM policy, but only if these do not exist yet. + +### Rule registry client + +The rule registry client can either be injected in the executor, or created in the scope of a request. It exposes a `search` method and a `bulkIndex` method. When `search` is called, it first gets all the rules the current user has access to, and adds these ids to the search request that it executes. This means that the user can only see data from rules they have access to. + +Both `search` and `bulkIndex` are fully typed, in the sense that they reflect the mappings defined for the registry. + +### Schema + +The following fields are available in the root rule registry: + +- `@timestamp`: the ISO timestamp of the alert event. For the lifecycle rule type helper, it is always the value of `startedAt` that is injected by the Kibana alerting framework. +- `event.kind`: signal (for the changeable alert document), state (for the state changes of the alert, e.g. when it opens, recovers, or changes in severity), or metric (individual evaluations that might be related to an alert). +- `event.action`: the reason for the event. This might be `open`, `close`, `active`, or `evaluate`. +- `tags`: tags attached to the alert. Right now they are copied over from the rule. +- `rule.id`: the identifier of the rule type, e.g. `apm.transaction_duration` +- `rule.uuid`: the saved objects id of the rule. +- `rule.name`: the name of the rule (as specified by the user). +- `rule.category`: the name of the rule type (as defined by the rule type producer) +- `kibana.rac.producer`: the producer of the rule type. Usually a Kibana plugin. e.g., `APM`. +- `kibana.rac.alert.id`: the id of the alert, that is unique within the context of the rule execution it was created in. E.g., for a rule that monitors latency for all services in all environments, this might be `opbeans-java:production`. +- `kibana.rac.alert.uuid`: the unique identifier for the alert during its lifespan. If an alert recovers (or closes), this identifier is re-generated when it is opened again. +- `kibana.rac.alert.status`: the status of the alert. Can be `open` or `closed`. +- `kibana.rac.alert.start`: the ISO timestamp of the time at which the alert started. +- `kibana.rac.alert.end`: the ISO timestamp of the time at which the alert recovered. +- `kibana.rac.alert.duration.us`: the duration of the alert, in microseconds. This is always the difference between either the current time, or the time when the alert recovered. +- `kibana.rac.alert.severity.level`: the severity of the alert, as a keyword (e.g. critical). +- `kibana.rac.alert.severity.value`: the severity of the alert, as a numerical value, which allows sorting. + +This list is not final - just a start. Field names might change or moved to a scoped registry. If we implement log and sequence based rule types the list of fields will grow. If a rule type needs additional fields, the recommendation would be to have the field in its own registry first (or in its producer’s registry), and if usage is more broadly adopted, it can be moved to the root registry. diff --git a/x-pack/plugins/rule_registry/common/index.ts b/x-pack/plugins/rule_registry/common/index.ts new file mode 100644 index 0000000000000..6cc0ccaa93a6d --- /dev/null +++ b/x-pack/plugins/rule_registry/common/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './types'; diff --git a/x-pack/plugins/rule_registry/common/types.ts b/x-pack/plugins/rule_registry/common/types.ts new file mode 100644 index 0000000000000..d0d15d86a2248 --- /dev/null +++ b/x-pack/plugins/rule_registry/common/types.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export enum AlertSeverityLevel { + warning = 'warning', + critical = 'critical', +} + +const alertSeverityLevelValues = { + [AlertSeverityLevel.warning]: 70, + [AlertSeverityLevel.critical]: 90, +}; + +export function getAlertSeverityLevelValue(level: AlertSeverityLevel) { + return alertSeverityLevelValues[level]; +} diff --git a/x-pack/plugins/rule_registry/jest.config.js b/x-pack/plugins/rule_registry/jest.config.js new file mode 100644 index 0000000000000..df8ac522e4b5d --- /dev/null +++ b/x-pack/plugins/rule_registry/jest.config.js @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +module.exports = { + preset: '@kbn/test', + rootDir: '../../..', + roots: ['/x-pack/plugins/rule_registry'], +}; diff --git a/x-pack/plugins/rule_registry/kibana.json b/x-pack/plugins/rule_registry/kibana.json new file mode 100644 index 0000000000000..dea6ef560cc2d --- /dev/null +++ b/x-pack/plugins/rule_registry/kibana.json @@ -0,0 +1,13 @@ +{ + "id": "ruleRegistry", + "version": "8.0.0", + "kibanaVersion": "kibana", + "configPath": [ + "xpack", + "ruleRegistry" + ], + "requiredPlugins": [ + "alerting" + ], + "server": true +} diff --git a/x-pack/plugins/rule_registry/scripts/generate_ecs_fieldmap/index.js b/x-pack/plugins/rule_registry/scripts/generate_ecs_fieldmap/index.js new file mode 100644 index 0000000000000..6e3a8f7cbe663 --- /dev/null +++ b/x-pack/plugins/rule_registry/scripts/generate_ecs_fieldmap/index.js @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +const path = require('path'); +const fs = require('fs'); +const util = require('util'); +const yaml = require('js-yaml'); +const { exec: execCb } = require('child_process'); +const { mapValues } = require('lodash'); + +const exists = util.promisify(fs.exists); +const readFile = util.promisify(fs.readFile); +const writeFile = util.promisify(fs.writeFile); +const mkdir = util.promisify(fs.mkdir); +const rmdir = util.promisify(fs.rmdir); +const exec = util.promisify(execCb); + +const ecsDir = path.resolve(__dirname, '../../../../../../ecs'); +const ecsTemplateFilename = path.join(ecsDir, 'generated/elasticsearch/7/template.json'); +const flatYamlFilename = path.join(ecsDir, 'generated/ecs/ecs_flat.yml'); + +const outputDir = path.join(__dirname, '../../server/generated'); + +const outputFieldMapFilename = path.join(outputDir, 'ecs_field_map.ts'); +const outputMappingFilename = path.join(outputDir, 'ecs_mappings.json'); + +async function generate() { + const allExists = await Promise.all([exists(ecsDir), exists(ecsTemplateFilename)]); + + if (!allExists.every(Boolean)) { + throw new Error( + `Directory not found: ${ecsDir} - did you checkout elastic/ecs as a peer of this repo?` + ); + } + + const [template, flatYaml] = await Promise.all([ + readFile(ecsTemplateFilename, { encoding: 'utf-8' }).then((str) => JSON.parse(str)), + (async () => yaml.safeLoad(await readFile(flatYamlFilename)))(), + ]); + + const mappings = { + properties: template.mappings.properties, + }; + + const fields = mapValues(flatYaml, (description) => { + return { + type: description.type, + array: description.normalize.includes('array'), + required: !!description.required, + }; + }); + + const hasOutputDir = await exists(outputDir); + + if (hasOutputDir) { + await rmdir(outputDir, { recursive: true }); + } + + await mkdir(outputDir); + + await Promise.all([ + writeFile( + outputFieldMapFilename, + ` + export const ecsFieldMap = ${JSON.stringify(fields, null, 2)} as const + `, + { encoding: 'utf-8' } + ).then(() => { + return exec(`node scripts/eslint --fix ${outputFieldMapFilename}`); + }), + writeFile(outputMappingFilename, JSON.stringify(mappings, null, 2)), + ]); +} + +generate().catch((err) => { + console.log(err); + process.exit(1); +}); diff --git a/x-pack/plugins/rule_registry/server/generated/ecs_field_map.ts b/x-pack/plugins/rule_registry/server/generated/ecs_field_map.ts new file mode 100644 index 0000000000000..cd8865a3f57c2 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/generated/ecs_field_map.ts @@ -0,0 +1,3374 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const ecsFieldMap = { + '@timestamp': { + type: 'date', + array: false, + required: true, + }, + 'agent.build.original': { + type: 'keyword', + array: false, + required: false, + }, + 'agent.ephemeral_id': { + type: 'keyword', + array: false, + required: false, + }, + 'agent.id': { + type: 'keyword', + array: false, + required: false, + }, + 'agent.name': { + type: 'keyword', + array: false, + required: false, + }, + 'agent.type': { + type: 'keyword', + array: false, + required: false, + }, + 'agent.version': { + type: 'keyword', + array: false, + required: false, + }, + 'client.address': { + type: 'keyword', + array: false, + required: false, + }, + 'client.as.number': { + type: 'long', + array: false, + required: false, + }, + 'client.as.organization.name': { + type: 'keyword', + array: false, + required: false, + }, + 'client.bytes': { + type: 'long', + array: false, + required: false, + }, + 'client.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'client.geo.city_name': { + type: 'keyword', + array: false, + required: false, + }, + 'client.geo.continent_name': { + type: 'keyword', + array: false, + required: false, + }, + 'client.geo.country_iso_code': { + type: 'keyword', + array: false, + required: false, + }, + 'client.geo.country_name': { + type: 'keyword', + array: false, + required: false, + }, + 'client.geo.location': { + type: 'geo_point', + array: false, + required: false, + }, + 'client.geo.name': { + type: 'keyword', + array: false, + required: false, + }, + 'client.geo.region_iso_code': { + type: 'keyword', + array: false, + required: false, + }, + 'client.geo.region_name': { + type: 'keyword', + array: false, + required: false, + }, + 'client.ip': { + type: 'ip', + array: false, + required: false, + }, + 'client.mac': { + type: 'keyword', + array: false, + required: false, + }, + 'client.nat.ip': { + type: 'ip', + array: false, + required: false, + }, + 'client.nat.port': { + type: 'long', + array: false, + required: false, + }, + 'client.packets': { + type: 'long', + array: false, + required: false, + }, + 'client.port': { + type: 'long', + array: false, + required: false, + }, + 'client.registered_domain': { + type: 'keyword', + array: false, + required: false, + }, + 'client.subdomain': { + type: 'keyword', + array: false, + required: false, + }, + 'client.top_level_domain': { + type: 'keyword', + array: false, + required: false, + }, + 'client.user.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'client.user.email': { + type: 'keyword', + array: false, + required: false, + }, + 'client.user.full_name': { + type: 'keyword', + array: false, + required: false, + }, + 'client.user.group.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'client.user.group.id': { + type: 'keyword', + array: false, + required: false, + }, + 'client.user.group.name': { + type: 'keyword', + array: false, + required: false, + }, + 'client.user.hash': { + type: 'keyword', + array: false, + required: false, + }, + 'client.user.id': { + type: 'keyword', + array: false, + required: false, + }, + 'client.user.name': { + type: 'keyword', + array: false, + required: false, + }, + 'client.user.roles': { + type: 'keyword', + array: true, + required: false, + }, + 'cloud.account.id': { + type: 'keyword', + array: false, + required: false, + }, + 'cloud.account.name': { + type: 'keyword', + array: false, + required: false, + }, + 'cloud.availability_zone': { + type: 'keyword', + array: false, + required: false, + }, + 'cloud.instance.id': { + type: 'keyword', + array: false, + required: false, + }, + 'cloud.instance.name': { + type: 'keyword', + array: false, + required: false, + }, + 'cloud.machine.type': { + type: 'keyword', + array: false, + required: false, + }, + 'cloud.project.id': { + type: 'keyword', + array: false, + required: false, + }, + 'cloud.project.name': { + type: 'keyword', + array: false, + required: false, + }, + 'cloud.provider': { + type: 'keyword', + array: false, + required: false, + }, + 'cloud.region': { + type: 'keyword', + array: false, + required: false, + }, + 'container.id': { + type: 'keyword', + array: false, + required: false, + }, + 'container.image.name': { + type: 'keyword', + array: false, + required: false, + }, + 'container.image.tag': { + type: 'keyword', + array: true, + required: false, + }, + 'container.labels': { + type: 'object', + array: false, + required: false, + }, + 'container.name': { + type: 'keyword', + array: false, + required: false, + }, + 'container.runtime': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.address': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.as.number': { + type: 'long', + array: false, + required: false, + }, + 'destination.as.organization.name': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.bytes': { + type: 'long', + array: false, + required: false, + }, + 'destination.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.geo.city_name': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.geo.continent_name': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.geo.country_iso_code': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.geo.country_name': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.geo.location': { + type: 'geo_point', + array: false, + required: false, + }, + 'destination.geo.name': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.geo.region_iso_code': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.geo.region_name': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.ip': { + type: 'ip', + array: false, + required: false, + }, + 'destination.mac': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.nat.ip': { + type: 'ip', + array: false, + required: false, + }, + 'destination.nat.port': { + type: 'long', + array: false, + required: false, + }, + 'destination.packets': { + type: 'long', + array: false, + required: false, + }, + 'destination.port': { + type: 'long', + array: false, + required: false, + }, + 'destination.registered_domain': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.subdomain': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.top_level_domain': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.user.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.user.email': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.user.full_name': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.user.group.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.user.group.id': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.user.group.name': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.user.hash': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.user.id': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.user.name': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.user.roles': { + type: 'keyword', + array: true, + required: false, + }, + 'dll.code_signature.exists': { + type: 'boolean', + array: false, + required: false, + }, + 'dll.code_signature.status': { + type: 'keyword', + array: false, + required: false, + }, + 'dll.code_signature.subject_name': { + type: 'keyword', + array: false, + required: false, + }, + 'dll.code_signature.trusted': { + type: 'boolean', + array: false, + required: false, + }, + 'dll.code_signature.valid': { + type: 'boolean', + array: false, + required: false, + }, + 'dll.hash.md5': { + type: 'keyword', + array: false, + required: false, + }, + 'dll.hash.sha1': { + type: 'keyword', + array: false, + required: false, + }, + 'dll.hash.sha256': { + type: 'keyword', + array: false, + required: false, + }, + 'dll.hash.sha512': { + type: 'keyword', + array: false, + required: false, + }, + 'dll.name': { + type: 'keyword', + array: false, + required: false, + }, + 'dll.path': { + type: 'keyword', + array: false, + required: false, + }, + 'dll.pe.architecture': { + type: 'keyword', + array: false, + required: false, + }, + 'dll.pe.company': { + type: 'keyword', + array: false, + required: false, + }, + 'dll.pe.description': { + type: 'keyword', + array: false, + required: false, + }, + 'dll.pe.file_version': { + type: 'keyword', + array: false, + required: false, + }, + 'dll.pe.imphash': { + type: 'keyword', + array: false, + required: false, + }, + 'dll.pe.original_file_name': { + type: 'keyword', + array: false, + required: false, + }, + 'dll.pe.product': { + type: 'keyword', + array: false, + required: false, + }, + 'dns.answers': { + type: 'object', + array: true, + required: false, + }, + 'dns.answers.class': { + type: 'keyword', + array: false, + required: false, + }, + 'dns.answers.data': { + type: 'keyword', + array: false, + required: false, + }, + 'dns.answers.name': { + type: 'keyword', + array: false, + required: false, + }, + 'dns.answers.ttl': { + type: 'long', + array: false, + required: false, + }, + 'dns.answers.type': { + type: 'keyword', + array: false, + required: false, + }, + 'dns.header_flags': { + type: 'keyword', + array: true, + required: false, + }, + 'dns.id': { + type: 'keyword', + array: false, + required: false, + }, + 'dns.op_code': { + type: 'keyword', + array: false, + required: false, + }, + 'dns.question.class': { + type: 'keyword', + array: false, + required: false, + }, + 'dns.question.name': { + type: 'keyword', + array: false, + required: false, + }, + 'dns.question.registered_domain': { + type: 'keyword', + array: false, + required: false, + }, + 'dns.question.subdomain': { + type: 'keyword', + array: false, + required: false, + }, + 'dns.question.top_level_domain': { + type: 'keyword', + array: false, + required: false, + }, + 'dns.question.type': { + type: 'keyword', + array: false, + required: false, + }, + 'dns.resolved_ip': { + type: 'ip', + array: true, + required: false, + }, + 'dns.response_code': { + type: 'keyword', + array: false, + required: false, + }, + 'dns.type': { + type: 'keyword', + array: false, + required: false, + }, + 'ecs.version': { + type: 'keyword', + array: false, + required: true, + }, + 'error.code': { + type: 'keyword', + array: false, + required: false, + }, + 'error.id': { + type: 'keyword', + array: false, + required: false, + }, + 'error.message': { + type: 'text', + array: false, + required: false, + }, + 'error.stack_trace': { + type: 'keyword', + array: false, + required: false, + }, + 'error.type': { + type: 'keyword', + array: false, + required: false, + }, + 'event.action': { + type: 'keyword', + array: false, + required: false, + }, + 'event.category': { + type: 'keyword', + array: true, + required: false, + }, + 'event.code': { + type: 'keyword', + array: false, + required: false, + }, + 'event.created': { + type: 'date', + array: false, + required: false, + }, + 'event.dataset': { + type: 'keyword', + array: false, + required: false, + }, + 'event.duration': { + type: 'long', + array: false, + required: false, + }, + 'event.end': { + type: 'date', + array: false, + required: false, + }, + 'event.hash': { + type: 'keyword', + array: false, + required: false, + }, + 'event.id': { + type: 'keyword', + array: false, + required: false, + }, + 'event.ingested': { + type: 'date', + array: false, + required: false, + }, + 'event.kind': { + type: 'keyword', + array: false, + required: false, + }, + 'event.module': { + type: 'keyword', + array: false, + required: false, + }, + 'event.original': { + type: 'keyword', + array: false, + required: false, + }, + 'event.outcome': { + type: 'keyword', + array: false, + required: false, + }, + 'event.provider': { + type: 'keyword', + array: false, + required: false, + }, + 'event.reason': { + type: 'keyword', + array: false, + required: false, + }, + 'event.reference': { + type: 'keyword', + array: false, + required: false, + }, + 'event.risk_score': { + type: 'float', + array: false, + required: false, + }, + 'event.risk_score_norm': { + type: 'float', + array: false, + required: false, + }, + 'event.sequence': { + type: 'long', + array: false, + required: false, + }, + 'event.severity': { + type: 'long', + array: false, + required: false, + }, + 'event.start': { + type: 'date', + array: false, + required: false, + }, + 'event.timezone': { + type: 'keyword', + array: false, + required: false, + }, + 'event.type': { + type: 'keyword', + array: true, + required: false, + }, + 'event.url': { + type: 'keyword', + array: false, + required: false, + }, + 'file.accessed': { + type: 'date', + array: false, + required: false, + }, + 'file.attributes': { + type: 'keyword', + array: true, + required: false, + }, + 'file.code_signature.exists': { + type: 'boolean', + array: false, + required: false, + }, + 'file.code_signature.status': { + type: 'keyword', + array: false, + required: false, + }, + 'file.code_signature.subject_name': { + type: 'keyword', + array: false, + required: false, + }, + 'file.code_signature.trusted': { + type: 'boolean', + array: false, + required: false, + }, + 'file.code_signature.valid': { + type: 'boolean', + array: false, + required: false, + }, + 'file.created': { + type: 'date', + array: false, + required: false, + }, + 'file.ctime': { + type: 'date', + array: false, + required: false, + }, + 'file.device': { + type: 'keyword', + array: false, + required: false, + }, + 'file.directory': { + type: 'keyword', + array: false, + required: false, + }, + 'file.drive_letter': { + type: 'keyword', + array: false, + required: false, + }, + 'file.extension': { + type: 'keyword', + array: false, + required: false, + }, + 'file.gid': { + type: 'keyword', + array: false, + required: false, + }, + 'file.group': { + type: 'keyword', + array: false, + required: false, + }, + 'file.hash.md5': { + type: 'keyword', + array: false, + required: false, + }, + 'file.hash.sha1': { + type: 'keyword', + array: false, + required: false, + }, + 'file.hash.sha256': { + type: 'keyword', + array: false, + required: false, + }, + 'file.hash.sha512': { + type: 'keyword', + array: false, + required: false, + }, + 'file.inode': { + type: 'keyword', + array: false, + required: false, + }, + 'file.mime_type': { + type: 'keyword', + array: false, + required: false, + }, + 'file.mode': { + type: 'keyword', + array: false, + required: false, + }, + 'file.mtime': { + type: 'date', + array: false, + required: false, + }, + 'file.name': { + type: 'keyword', + array: false, + required: false, + }, + 'file.owner': { + type: 'keyword', + array: false, + required: false, + }, + 'file.path': { + type: 'keyword', + array: false, + required: false, + }, + 'file.pe.architecture': { + type: 'keyword', + array: false, + required: false, + }, + 'file.pe.company': { + type: 'keyword', + array: false, + required: false, + }, + 'file.pe.description': { + type: 'keyword', + array: false, + required: false, + }, + 'file.pe.file_version': { + type: 'keyword', + array: false, + required: false, + }, + 'file.pe.imphash': { + type: 'keyword', + array: false, + required: false, + }, + 'file.pe.original_file_name': { + type: 'keyword', + array: false, + required: false, + }, + 'file.pe.product': { + type: 'keyword', + array: false, + required: false, + }, + 'file.size': { + type: 'long', + array: false, + required: false, + }, + 'file.target_path': { + type: 'keyword', + array: false, + required: false, + }, + 'file.type': { + type: 'keyword', + array: false, + required: false, + }, + 'file.uid': { + type: 'keyword', + array: false, + required: false, + }, + 'file.x509.alternative_names': { + type: 'keyword', + array: true, + required: false, + }, + 'file.x509.issuer.common_name': { + type: 'keyword', + array: true, + required: false, + }, + 'file.x509.issuer.country': { + type: 'keyword', + array: true, + required: false, + }, + 'file.x509.issuer.distinguished_name': { + type: 'keyword', + array: false, + required: false, + }, + 'file.x509.issuer.locality': { + type: 'keyword', + array: true, + required: false, + }, + 'file.x509.issuer.organization': { + type: 'keyword', + array: true, + required: false, + }, + 'file.x509.issuer.organizational_unit': { + type: 'keyword', + array: true, + required: false, + }, + 'file.x509.issuer.state_or_province': { + type: 'keyword', + array: true, + required: false, + }, + 'file.x509.not_after': { + type: 'date', + array: false, + required: false, + }, + 'file.x509.not_before': { + type: 'date', + array: false, + required: false, + }, + 'file.x509.public_key_algorithm': { + type: 'keyword', + array: false, + required: false, + }, + 'file.x509.public_key_curve': { + type: 'keyword', + array: false, + required: false, + }, + 'file.x509.public_key_exponent': { + type: 'long', + array: false, + required: false, + }, + 'file.x509.public_key_size': { + type: 'long', + array: false, + required: false, + }, + 'file.x509.serial_number': { + type: 'keyword', + array: false, + required: false, + }, + 'file.x509.signature_algorithm': { + type: 'keyword', + array: false, + required: false, + }, + 'file.x509.subject.common_name': { + type: 'keyword', + array: true, + required: false, + }, + 'file.x509.subject.country': { + type: 'keyword', + array: true, + required: false, + }, + 'file.x509.subject.distinguished_name': { + type: 'keyword', + array: false, + required: false, + }, + 'file.x509.subject.locality': { + type: 'keyword', + array: true, + required: false, + }, + 'file.x509.subject.organization': { + type: 'keyword', + array: true, + required: false, + }, + 'file.x509.subject.organizational_unit': { + type: 'keyword', + array: true, + required: false, + }, + 'file.x509.subject.state_or_province': { + type: 'keyword', + array: true, + required: false, + }, + 'file.x509.version_number': { + type: 'keyword', + array: false, + required: false, + }, + 'group.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'group.id': { + type: 'keyword', + array: false, + required: false, + }, + 'group.name': { + type: 'keyword', + array: false, + required: false, + }, + 'host.architecture': { + type: 'keyword', + array: false, + required: false, + }, + 'host.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'host.geo.city_name': { + type: 'keyword', + array: false, + required: false, + }, + 'host.geo.continent_name': { + type: 'keyword', + array: false, + required: false, + }, + 'host.geo.country_iso_code': { + type: 'keyword', + array: false, + required: false, + }, + 'host.geo.country_name': { + type: 'keyword', + array: false, + required: false, + }, + 'host.geo.location': { + type: 'geo_point', + array: false, + required: false, + }, + 'host.geo.name': { + type: 'keyword', + array: false, + required: false, + }, + 'host.geo.region_iso_code': { + type: 'keyword', + array: false, + required: false, + }, + 'host.geo.region_name': { + type: 'keyword', + array: false, + required: false, + }, + 'host.hostname': { + type: 'keyword', + array: false, + required: false, + }, + 'host.id': { + type: 'keyword', + array: false, + required: false, + }, + 'host.ip': { + type: 'ip', + array: true, + required: false, + }, + 'host.mac': { + type: 'keyword', + array: true, + required: false, + }, + 'host.name': { + type: 'keyword', + array: false, + required: false, + }, + 'host.os.family': { + type: 'keyword', + array: false, + required: false, + }, + 'host.os.full': { + type: 'keyword', + array: false, + required: false, + }, + 'host.os.kernel': { + type: 'keyword', + array: false, + required: false, + }, + 'host.os.name': { + type: 'keyword', + array: false, + required: false, + }, + 'host.os.platform': { + type: 'keyword', + array: false, + required: false, + }, + 'host.os.type': { + type: 'keyword', + array: false, + required: false, + }, + 'host.os.version': { + type: 'keyword', + array: false, + required: false, + }, + 'host.type': { + type: 'keyword', + array: false, + required: false, + }, + 'host.uptime': { + type: 'long', + array: false, + required: false, + }, + 'host.user.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'host.user.email': { + type: 'keyword', + array: false, + required: false, + }, + 'host.user.full_name': { + type: 'keyword', + array: false, + required: false, + }, + 'host.user.group.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'host.user.group.id': { + type: 'keyword', + array: false, + required: false, + }, + 'host.user.group.name': { + type: 'keyword', + array: false, + required: false, + }, + 'host.user.hash': { + type: 'keyword', + array: false, + required: false, + }, + 'host.user.id': { + type: 'keyword', + array: false, + required: false, + }, + 'host.user.name': { + type: 'keyword', + array: false, + required: false, + }, + 'host.user.roles': { + type: 'keyword', + array: true, + required: false, + }, + 'http.request.body.bytes': { + type: 'long', + array: false, + required: false, + }, + 'http.request.body.content': { + type: 'keyword', + array: false, + required: false, + }, + 'http.request.bytes': { + type: 'long', + array: false, + required: false, + }, + 'http.request.method': { + type: 'keyword', + array: false, + required: false, + }, + 'http.request.mime_type': { + type: 'keyword', + array: false, + required: false, + }, + 'http.request.referrer': { + type: 'keyword', + array: false, + required: false, + }, + 'http.response.body.bytes': { + type: 'long', + array: false, + required: false, + }, + 'http.response.body.content': { + type: 'keyword', + array: false, + required: false, + }, + 'http.response.bytes': { + type: 'long', + array: false, + required: false, + }, + 'http.response.mime_type': { + type: 'keyword', + array: false, + required: false, + }, + 'http.response.status_code': { + type: 'long', + array: false, + required: false, + }, + 'http.version': { + type: 'keyword', + array: false, + required: false, + }, + labels: { + type: 'object', + array: false, + required: false, + }, + 'log.file.path': { + type: 'keyword', + array: false, + required: false, + }, + 'log.level': { + type: 'keyword', + array: false, + required: false, + }, + 'log.logger': { + type: 'keyword', + array: false, + required: false, + }, + 'log.origin.file.line': { + type: 'integer', + array: false, + required: false, + }, + 'log.origin.file.name': { + type: 'keyword', + array: false, + required: false, + }, + 'log.origin.function': { + type: 'keyword', + array: false, + required: false, + }, + 'log.original': { + type: 'keyword', + array: false, + required: false, + }, + 'log.syslog': { + type: 'object', + array: false, + required: false, + }, + 'log.syslog.facility.code': { + type: 'long', + array: false, + required: false, + }, + 'log.syslog.facility.name': { + type: 'keyword', + array: false, + required: false, + }, + 'log.syslog.priority': { + type: 'long', + array: false, + required: false, + }, + 'log.syslog.severity.code': { + type: 'long', + array: false, + required: false, + }, + 'log.syslog.severity.name': { + type: 'keyword', + array: false, + required: false, + }, + message: { + type: 'text', + array: false, + required: false, + }, + 'network.application': { + type: 'keyword', + array: false, + required: false, + }, + 'network.bytes': { + type: 'long', + array: false, + required: false, + }, + 'network.community_id': { + type: 'keyword', + array: false, + required: false, + }, + 'network.direction': { + type: 'keyword', + array: false, + required: false, + }, + 'network.forwarded_ip': { + type: 'ip', + array: false, + required: false, + }, + 'network.iana_number': { + type: 'keyword', + array: false, + required: false, + }, + 'network.inner': { + type: 'object', + array: false, + required: false, + }, + 'network.inner.vlan.id': { + type: 'keyword', + array: false, + required: false, + }, + 'network.inner.vlan.name': { + type: 'keyword', + array: false, + required: false, + }, + 'network.name': { + type: 'keyword', + array: false, + required: false, + }, + 'network.packets': { + type: 'long', + array: false, + required: false, + }, + 'network.protocol': { + type: 'keyword', + array: false, + required: false, + }, + 'network.transport': { + type: 'keyword', + array: false, + required: false, + }, + 'network.type': { + type: 'keyword', + array: false, + required: false, + }, + 'network.vlan.id': { + type: 'keyword', + array: false, + required: false, + }, + 'network.vlan.name': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.egress': { + type: 'object', + array: false, + required: false, + }, + 'observer.egress.interface.alias': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.egress.interface.id': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.egress.interface.name': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.egress.vlan.id': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.egress.vlan.name': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.egress.zone': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.geo.city_name': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.geo.continent_name': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.geo.country_iso_code': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.geo.country_name': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.geo.location': { + type: 'geo_point', + array: false, + required: false, + }, + 'observer.geo.name': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.geo.region_iso_code': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.geo.region_name': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.hostname': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.ingress': { + type: 'object', + array: false, + required: false, + }, + 'observer.ingress.interface.alias': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.ingress.interface.id': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.ingress.interface.name': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.ingress.vlan.id': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.ingress.vlan.name': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.ingress.zone': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.ip': { + type: 'ip', + array: true, + required: false, + }, + 'observer.mac': { + type: 'keyword', + array: true, + required: false, + }, + 'observer.name': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.os.family': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.os.full': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.os.kernel': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.os.name': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.os.platform': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.os.type': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.os.version': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.product': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.serial_number': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.type': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.vendor': { + type: 'keyword', + array: false, + required: false, + }, + 'observer.version': { + type: 'keyword', + array: false, + required: false, + }, + 'organization.id': { + type: 'keyword', + array: false, + required: false, + }, + 'organization.name': { + type: 'keyword', + array: false, + required: false, + }, + 'package.architecture': { + type: 'keyword', + array: false, + required: false, + }, + 'package.build_version': { + type: 'keyword', + array: false, + required: false, + }, + 'package.checksum': { + type: 'keyword', + array: false, + required: false, + }, + 'package.description': { + type: 'keyword', + array: false, + required: false, + }, + 'package.install_scope': { + type: 'keyword', + array: false, + required: false, + }, + 'package.installed': { + type: 'date', + array: false, + required: false, + }, + 'package.license': { + type: 'keyword', + array: false, + required: false, + }, + 'package.name': { + type: 'keyword', + array: false, + required: false, + }, + 'package.path': { + type: 'keyword', + array: false, + required: false, + }, + 'package.reference': { + type: 'keyword', + array: false, + required: false, + }, + 'package.size': { + type: 'long', + array: false, + required: false, + }, + 'package.type': { + type: 'keyword', + array: false, + required: false, + }, + 'package.version': { + type: 'keyword', + array: false, + required: false, + }, + 'process.args': { + type: 'keyword', + array: true, + required: false, + }, + 'process.args_count': { + type: 'long', + array: false, + required: false, + }, + 'process.code_signature.exists': { + type: 'boolean', + array: false, + required: false, + }, + 'process.code_signature.status': { + type: 'keyword', + array: false, + required: false, + }, + 'process.code_signature.subject_name': { + type: 'keyword', + array: false, + required: false, + }, + 'process.code_signature.trusted': { + type: 'boolean', + array: false, + required: false, + }, + 'process.code_signature.valid': { + type: 'boolean', + array: false, + required: false, + }, + 'process.command_line': { + type: 'keyword', + array: false, + required: false, + }, + 'process.entity_id': { + type: 'keyword', + array: false, + required: false, + }, + 'process.executable': { + type: 'keyword', + array: false, + required: false, + }, + 'process.exit_code': { + type: 'long', + array: false, + required: false, + }, + 'process.hash.md5': { + type: 'keyword', + array: false, + required: false, + }, + 'process.hash.sha1': { + type: 'keyword', + array: false, + required: false, + }, + 'process.hash.sha256': { + type: 'keyword', + array: false, + required: false, + }, + 'process.hash.sha512': { + type: 'keyword', + array: false, + required: false, + }, + 'process.name': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.args': { + type: 'keyword', + array: true, + required: false, + }, + 'process.parent.args_count': { + type: 'long', + array: false, + required: false, + }, + 'process.parent.code_signature.exists': { + type: 'boolean', + array: false, + required: false, + }, + 'process.parent.code_signature.status': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.code_signature.subject_name': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.code_signature.trusted': { + type: 'boolean', + array: false, + required: false, + }, + 'process.parent.code_signature.valid': { + type: 'boolean', + array: false, + required: false, + }, + 'process.parent.command_line': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.entity_id': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.executable': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.exit_code': { + type: 'long', + array: false, + required: false, + }, + 'process.parent.hash.md5': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.hash.sha1': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.hash.sha256': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.hash.sha512': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.name': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.pe.architecture': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.pe.company': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.pe.description': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.pe.file_version': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.pe.imphash': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.pe.original_file_name': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.pe.product': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.pgid': { + type: 'long', + array: false, + required: false, + }, + 'process.parent.pid': { + type: 'long', + array: false, + required: false, + }, + 'process.parent.ppid': { + type: 'long', + array: false, + required: false, + }, + 'process.parent.start': { + type: 'date', + array: false, + required: false, + }, + 'process.parent.thread.id': { + type: 'long', + array: false, + required: false, + }, + 'process.parent.thread.name': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.title': { + type: 'keyword', + array: false, + required: false, + }, + 'process.parent.uptime': { + type: 'long', + array: false, + required: false, + }, + 'process.parent.working_directory': { + type: 'keyword', + array: false, + required: false, + }, + 'process.pe.architecture': { + type: 'keyword', + array: false, + required: false, + }, + 'process.pe.company': { + type: 'keyword', + array: false, + required: false, + }, + 'process.pe.description': { + type: 'keyword', + array: false, + required: false, + }, + 'process.pe.file_version': { + type: 'keyword', + array: false, + required: false, + }, + 'process.pe.imphash': { + type: 'keyword', + array: false, + required: false, + }, + 'process.pe.original_file_name': { + type: 'keyword', + array: false, + required: false, + }, + 'process.pe.product': { + type: 'keyword', + array: false, + required: false, + }, + 'process.pgid': { + type: 'long', + array: false, + required: false, + }, + 'process.pid': { + type: 'long', + array: false, + required: false, + }, + 'process.ppid': { + type: 'long', + array: false, + required: false, + }, + 'process.start': { + type: 'date', + array: false, + required: false, + }, + 'process.thread.id': { + type: 'long', + array: false, + required: false, + }, + 'process.thread.name': { + type: 'keyword', + array: false, + required: false, + }, + 'process.title': { + type: 'keyword', + array: false, + required: false, + }, + 'process.uptime': { + type: 'long', + array: false, + required: false, + }, + 'process.working_directory': { + type: 'keyword', + array: false, + required: false, + }, + 'registry.data.bytes': { + type: 'keyword', + array: false, + required: false, + }, + 'registry.data.strings': { + type: 'keyword', + array: true, + required: false, + }, + 'registry.data.type': { + type: 'keyword', + array: false, + required: false, + }, + 'registry.hive': { + type: 'keyword', + array: false, + required: false, + }, + 'registry.key': { + type: 'keyword', + array: false, + required: false, + }, + 'registry.path': { + type: 'keyword', + array: false, + required: false, + }, + 'registry.value': { + type: 'keyword', + array: false, + required: false, + }, + 'related.hash': { + type: 'keyword', + array: true, + required: false, + }, + 'related.hosts': { + type: 'keyword', + array: true, + required: false, + }, + 'related.ip': { + type: 'ip', + array: true, + required: false, + }, + 'related.user': { + type: 'keyword', + array: true, + required: false, + }, + 'rule.author': { + type: 'keyword', + array: true, + required: false, + }, + 'rule.category': { + type: 'keyword', + array: false, + required: false, + }, + 'rule.description': { + type: 'keyword', + array: false, + required: false, + }, + 'rule.id': { + type: 'keyword', + array: false, + required: false, + }, + 'rule.license': { + type: 'keyword', + array: false, + required: false, + }, + 'rule.name': { + type: 'keyword', + array: false, + required: false, + }, + 'rule.reference': { + type: 'keyword', + array: false, + required: false, + }, + 'rule.ruleset': { + type: 'keyword', + array: false, + required: false, + }, + 'rule.uuid': { + type: 'keyword', + array: false, + required: false, + }, + 'rule.version': { + type: 'keyword', + array: false, + required: false, + }, + 'server.address': { + type: 'keyword', + array: false, + required: false, + }, + 'server.as.number': { + type: 'long', + array: false, + required: false, + }, + 'server.as.organization.name': { + type: 'keyword', + array: false, + required: false, + }, + 'server.bytes': { + type: 'long', + array: false, + required: false, + }, + 'server.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'server.geo.city_name': { + type: 'keyword', + array: false, + required: false, + }, + 'server.geo.continent_name': { + type: 'keyword', + array: false, + required: false, + }, + 'server.geo.country_iso_code': { + type: 'keyword', + array: false, + required: false, + }, + 'server.geo.country_name': { + type: 'keyword', + array: false, + required: false, + }, + 'server.geo.location': { + type: 'geo_point', + array: false, + required: false, + }, + 'server.geo.name': { + type: 'keyword', + array: false, + required: false, + }, + 'server.geo.region_iso_code': { + type: 'keyword', + array: false, + required: false, + }, + 'server.geo.region_name': { + type: 'keyword', + array: false, + required: false, + }, + 'server.ip': { + type: 'ip', + array: false, + required: false, + }, + 'server.mac': { + type: 'keyword', + array: false, + required: false, + }, + 'server.nat.ip': { + type: 'ip', + array: false, + required: false, + }, + 'server.nat.port': { + type: 'long', + array: false, + required: false, + }, + 'server.packets': { + type: 'long', + array: false, + required: false, + }, + 'server.port': { + type: 'long', + array: false, + required: false, + }, + 'server.registered_domain': { + type: 'keyword', + array: false, + required: false, + }, + 'server.subdomain': { + type: 'keyword', + array: false, + required: false, + }, + 'server.top_level_domain': { + type: 'keyword', + array: false, + required: false, + }, + 'server.user.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'server.user.email': { + type: 'keyword', + array: false, + required: false, + }, + 'server.user.full_name': { + type: 'keyword', + array: false, + required: false, + }, + 'server.user.group.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'server.user.group.id': { + type: 'keyword', + array: false, + required: false, + }, + 'server.user.group.name': { + type: 'keyword', + array: false, + required: false, + }, + 'server.user.hash': { + type: 'keyword', + array: false, + required: false, + }, + 'server.user.id': { + type: 'keyword', + array: false, + required: false, + }, + 'server.user.name': { + type: 'keyword', + array: false, + required: false, + }, + 'server.user.roles': { + type: 'keyword', + array: true, + required: false, + }, + 'service.ephemeral_id': { + type: 'keyword', + array: false, + required: false, + }, + 'service.id': { + type: 'keyword', + array: false, + required: false, + }, + 'service.name': { + type: 'keyword', + array: false, + required: false, + }, + 'service.node.name': { + type: 'keyword', + array: false, + required: false, + }, + 'service.state': { + type: 'keyword', + array: false, + required: false, + }, + 'service.type': { + type: 'keyword', + array: false, + required: false, + }, + 'service.version': { + type: 'keyword', + array: false, + required: false, + }, + 'source.address': { + type: 'keyword', + array: false, + required: false, + }, + 'source.as.number': { + type: 'long', + array: false, + required: false, + }, + 'source.as.organization.name': { + type: 'keyword', + array: false, + required: false, + }, + 'source.bytes': { + type: 'long', + array: false, + required: false, + }, + 'source.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'source.geo.city_name': { + type: 'keyword', + array: false, + required: false, + }, + 'source.geo.continent_name': { + type: 'keyword', + array: false, + required: false, + }, + 'source.geo.country_iso_code': { + type: 'keyword', + array: false, + required: false, + }, + 'source.geo.country_name': { + type: 'keyword', + array: false, + required: false, + }, + 'source.geo.location': { + type: 'geo_point', + array: false, + required: false, + }, + 'source.geo.name': { + type: 'keyword', + array: false, + required: false, + }, + 'source.geo.region_iso_code': { + type: 'keyword', + array: false, + required: false, + }, + 'source.geo.region_name': { + type: 'keyword', + array: false, + required: false, + }, + 'source.ip': { + type: 'ip', + array: false, + required: false, + }, + 'source.mac': { + type: 'keyword', + array: false, + required: false, + }, + 'source.nat.ip': { + type: 'ip', + array: false, + required: false, + }, + 'source.nat.port': { + type: 'long', + array: false, + required: false, + }, + 'source.packets': { + type: 'long', + array: false, + required: false, + }, + 'source.port': { + type: 'long', + array: false, + required: false, + }, + 'source.registered_domain': { + type: 'keyword', + array: false, + required: false, + }, + 'source.subdomain': { + type: 'keyword', + array: false, + required: false, + }, + 'source.top_level_domain': { + type: 'keyword', + array: false, + required: false, + }, + 'source.user.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'source.user.email': { + type: 'keyword', + array: false, + required: false, + }, + 'source.user.full_name': { + type: 'keyword', + array: false, + required: false, + }, + 'source.user.group.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'source.user.group.id': { + type: 'keyword', + array: false, + required: false, + }, + 'source.user.group.name': { + type: 'keyword', + array: false, + required: false, + }, + 'source.user.hash': { + type: 'keyword', + array: false, + required: false, + }, + 'source.user.id': { + type: 'keyword', + array: false, + required: false, + }, + 'source.user.name': { + type: 'keyword', + array: false, + required: false, + }, + 'source.user.roles': { + type: 'keyword', + array: true, + required: false, + }, + 'span.id': { + type: 'keyword', + array: false, + required: false, + }, + tags: { + type: 'keyword', + array: true, + required: false, + }, + 'threat.framework': { + type: 'keyword', + array: false, + required: false, + }, + 'threat.tactic.id': { + type: 'keyword', + array: true, + required: false, + }, + 'threat.tactic.name': { + type: 'keyword', + array: true, + required: false, + }, + 'threat.tactic.reference': { + type: 'keyword', + array: true, + required: false, + }, + 'threat.technique.id': { + type: 'keyword', + array: true, + required: false, + }, + 'threat.technique.name': { + type: 'keyword', + array: true, + required: false, + }, + 'threat.technique.reference': { + type: 'keyword', + array: true, + required: false, + }, + 'threat.technique.subtechnique.id': { + type: 'keyword', + array: true, + required: false, + }, + 'threat.technique.subtechnique.name': { + type: 'keyword', + array: true, + required: false, + }, + 'threat.technique.subtechnique.reference': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.cipher': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.client.certificate': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.client.certificate_chain': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.client.hash.md5': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.client.hash.sha1': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.client.hash.sha256': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.client.issuer': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.client.ja3': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.client.not_after': { + type: 'date', + array: false, + required: false, + }, + 'tls.client.not_before': { + type: 'date', + array: false, + required: false, + }, + 'tls.client.server_name': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.client.subject': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.client.supported_ciphers': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.client.x509.alternative_names': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.client.x509.issuer.common_name': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.client.x509.issuer.country': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.client.x509.issuer.distinguished_name': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.client.x509.issuer.locality': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.client.x509.issuer.organization': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.client.x509.issuer.organizational_unit': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.client.x509.issuer.state_or_province': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.client.x509.not_after': { + type: 'date', + array: false, + required: false, + }, + 'tls.client.x509.not_before': { + type: 'date', + array: false, + required: false, + }, + 'tls.client.x509.public_key_algorithm': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.client.x509.public_key_curve': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.client.x509.public_key_exponent': { + type: 'long', + array: false, + required: false, + }, + 'tls.client.x509.public_key_size': { + type: 'long', + array: false, + required: false, + }, + 'tls.client.x509.serial_number': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.client.x509.signature_algorithm': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.client.x509.subject.common_name': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.client.x509.subject.country': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.client.x509.subject.distinguished_name': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.client.x509.subject.locality': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.client.x509.subject.organization': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.client.x509.subject.organizational_unit': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.client.x509.subject.state_or_province': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.client.x509.version_number': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.curve': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.established': { + type: 'boolean', + array: false, + required: false, + }, + 'tls.next_protocol': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.resumed': { + type: 'boolean', + array: false, + required: false, + }, + 'tls.server.certificate': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.server.certificate_chain': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.server.hash.md5': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.server.hash.sha1': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.server.hash.sha256': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.server.issuer': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.server.ja3s': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.server.not_after': { + type: 'date', + array: false, + required: false, + }, + 'tls.server.not_before': { + type: 'date', + array: false, + required: false, + }, + 'tls.server.subject': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.server.x509.alternative_names': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.server.x509.issuer.common_name': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.server.x509.issuer.country': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.server.x509.issuer.distinguished_name': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.server.x509.issuer.locality': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.server.x509.issuer.organization': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.server.x509.issuer.organizational_unit': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.server.x509.issuer.state_or_province': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.server.x509.not_after': { + type: 'date', + array: false, + required: false, + }, + 'tls.server.x509.not_before': { + type: 'date', + array: false, + required: false, + }, + 'tls.server.x509.public_key_algorithm': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.server.x509.public_key_curve': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.server.x509.public_key_exponent': { + type: 'long', + array: false, + required: false, + }, + 'tls.server.x509.public_key_size': { + type: 'long', + array: false, + required: false, + }, + 'tls.server.x509.serial_number': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.server.x509.signature_algorithm': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.server.x509.subject.common_name': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.server.x509.subject.country': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.server.x509.subject.distinguished_name': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.server.x509.subject.locality': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.server.x509.subject.organization': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.server.x509.subject.organizational_unit': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.server.x509.subject.state_or_province': { + type: 'keyword', + array: true, + required: false, + }, + 'tls.server.x509.version_number': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.version': { + type: 'keyword', + array: false, + required: false, + }, + 'tls.version_protocol': { + type: 'keyword', + array: false, + required: false, + }, + 'trace.id': { + type: 'keyword', + array: false, + required: false, + }, + 'transaction.id': { + type: 'keyword', + array: false, + required: false, + }, + 'url.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'url.extension': { + type: 'keyword', + array: false, + required: false, + }, + 'url.fragment': { + type: 'keyword', + array: false, + required: false, + }, + 'url.full': { + type: 'keyword', + array: false, + required: false, + }, + 'url.original': { + type: 'keyword', + array: false, + required: false, + }, + 'url.password': { + type: 'keyword', + array: false, + required: false, + }, + 'url.path': { + type: 'keyword', + array: false, + required: false, + }, + 'url.port': { + type: 'long', + array: false, + required: false, + }, + 'url.query': { + type: 'keyword', + array: false, + required: false, + }, + 'url.registered_domain': { + type: 'keyword', + array: false, + required: false, + }, + 'url.scheme': { + type: 'keyword', + array: false, + required: false, + }, + 'url.subdomain': { + type: 'keyword', + array: false, + required: false, + }, + 'url.top_level_domain': { + type: 'keyword', + array: false, + required: false, + }, + 'url.username': { + type: 'keyword', + array: false, + required: false, + }, + 'user.changes.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'user.changes.email': { + type: 'keyword', + array: false, + required: false, + }, + 'user.changes.full_name': { + type: 'keyword', + array: false, + required: false, + }, + 'user.changes.group.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'user.changes.group.id': { + type: 'keyword', + array: false, + required: false, + }, + 'user.changes.group.name': { + type: 'keyword', + array: false, + required: false, + }, + 'user.changes.hash': { + type: 'keyword', + array: false, + required: false, + }, + 'user.changes.id': { + type: 'keyword', + array: false, + required: false, + }, + 'user.changes.name': { + type: 'keyword', + array: false, + required: false, + }, + 'user.changes.roles': { + type: 'keyword', + array: true, + required: false, + }, + 'user.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'user.effective.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'user.effective.email': { + type: 'keyword', + array: false, + required: false, + }, + 'user.effective.full_name': { + type: 'keyword', + array: false, + required: false, + }, + 'user.effective.group.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'user.effective.group.id': { + type: 'keyword', + array: false, + required: false, + }, + 'user.effective.group.name': { + type: 'keyword', + array: false, + required: false, + }, + 'user.effective.hash': { + type: 'keyword', + array: false, + required: false, + }, + 'user.effective.id': { + type: 'keyword', + array: false, + required: false, + }, + 'user.effective.name': { + type: 'keyword', + array: false, + required: false, + }, + 'user.effective.roles': { + type: 'keyword', + array: true, + required: false, + }, + 'user.email': { + type: 'keyword', + array: false, + required: false, + }, + 'user.full_name': { + type: 'keyword', + array: false, + required: false, + }, + 'user.group.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'user.group.id': { + type: 'keyword', + array: false, + required: false, + }, + 'user.group.name': { + type: 'keyword', + array: false, + required: false, + }, + 'user.hash': { + type: 'keyword', + array: false, + required: false, + }, + 'user.id': { + type: 'keyword', + array: false, + required: false, + }, + 'user.name': { + type: 'keyword', + array: false, + required: false, + }, + 'user.roles': { + type: 'keyword', + array: true, + required: false, + }, + 'user.target.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'user.target.email': { + type: 'keyword', + array: false, + required: false, + }, + 'user.target.full_name': { + type: 'keyword', + array: false, + required: false, + }, + 'user.target.group.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'user.target.group.id': { + type: 'keyword', + array: false, + required: false, + }, + 'user.target.group.name': { + type: 'keyword', + array: false, + required: false, + }, + 'user.target.hash': { + type: 'keyword', + array: false, + required: false, + }, + 'user.target.id': { + type: 'keyword', + array: false, + required: false, + }, + 'user.target.name': { + type: 'keyword', + array: false, + required: false, + }, + 'user.target.roles': { + type: 'keyword', + array: true, + required: false, + }, + 'user_agent.device.name': { + type: 'keyword', + array: false, + required: false, + }, + 'user_agent.name': { + type: 'keyword', + array: false, + required: false, + }, + 'user_agent.original': { + type: 'keyword', + array: false, + required: false, + }, + 'user_agent.os.family': { + type: 'keyword', + array: false, + required: false, + }, + 'user_agent.os.full': { + type: 'keyword', + array: false, + required: false, + }, + 'user_agent.os.kernel': { + type: 'keyword', + array: false, + required: false, + }, + 'user_agent.os.name': { + type: 'keyword', + array: false, + required: false, + }, + 'user_agent.os.platform': { + type: 'keyword', + array: false, + required: false, + }, + 'user_agent.os.type': { + type: 'keyword', + array: false, + required: false, + }, + 'user_agent.os.version': { + type: 'keyword', + array: false, + required: false, + }, + 'user_agent.version': { + type: 'keyword', + array: false, + required: false, + }, + 'vulnerability.category': { + type: 'keyword', + array: true, + required: false, + }, + 'vulnerability.classification': { + type: 'keyword', + array: false, + required: false, + }, + 'vulnerability.description': { + type: 'keyword', + array: false, + required: false, + }, + 'vulnerability.enumeration': { + type: 'keyword', + array: false, + required: false, + }, + 'vulnerability.id': { + type: 'keyword', + array: false, + required: false, + }, + 'vulnerability.reference': { + type: 'keyword', + array: false, + required: false, + }, + 'vulnerability.report_id': { + type: 'keyword', + array: false, + required: false, + }, + 'vulnerability.scanner.vendor': { + type: 'keyword', + array: false, + required: false, + }, + 'vulnerability.score.base': { + type: 'float', + array: false, + required: false, + }, + 'vulnerability.score.environmental': { + type: 'float', + array: false, + required: false, + }, + 'vulnerability.score.temporal': { + type: 'float', + array: false, + required: false, + }, + 'vulnerability.score.version': { + type: 'keyword', + array: false, + required: false, + }, + 'vulnerability.severity': { + type: 'keyword', + array: false, + required: false, + }, +} as const; diff --git a/x-pack/plugins/rule_registry/server/generated/ecs_mappings.json b/x-pack/plugins/rule_registry/server/generated/ecs_mappings.json new file mode 100644 index 0000000000000..f7cbfc3dfaae3 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/generated/ecs_mappings.json @@ -0,0 +1,3416 @@ +{ + "properties": { + "@timestamp": { + "type": "date" + }, + "agent": { + "properties": { + "build": { + "properties": { + "original": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ephemeral_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "client": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "as": { + "properties": { + "number": { + "type": "long" + }, + "organization": { + "properties": { + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "bytes": { + "type": "long" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "nat": { + "properties": { + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + } + } + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + }, + "registered_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "subdomain": { + "ignore_above": 1024, + "type": "keyword" + }, + "top_level_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "roles": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "cloud": { + "properties": { + "account": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "availability_zone": { + "ignore_above": 1024, + "type": "keyword" + }, + "instance": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "machine": { + "properties": { + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "project": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "provider": { + "ignore_above": 1024, + "type": "keyword" + }, + "region": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "container": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "image": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "tag": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "labels": { + "type": "object" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "runtime": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "destination": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "as": { + "properties": { + "number": { + "type": "long" + }, + "organization": { + "properties": { + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "bytes": { + "type": "long" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "nat": { + "properties": { + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + } + } + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + }, + "registered_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "subdomain": { + "ignore_above": 1024, + "type": "keyword" + }, + "top_level_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "roles": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "dll": { + "properties": { + "code_signature": { + "properties": { + "exists": { + "type": "boolean" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "trusted": { + "type": "boolean" + }, + "valid": { + "type": "boolean" + } + } + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha512": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "pe": { + "properties": { + "architecture": { + "ignore_above": 1024, + "type": "keyword" + }, + "company": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "file_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "imphash": { + "ignore_above": 1024, + "type": "keyword" + }, + "original_file_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "product": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "dns": { + "properties": { + "answers": { + "properties": { + "class": { + "ignore_above": 1024, + "type": "keyword" + }, + "data": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "ttl": { + "type": "long" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + }, + "type": "object" + }, + "header_flags": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "op_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "question": { + "properties": { + "class": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "registered_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "subdomain": { + "ignore_above": 1024, + "type": "keyword" + }, + "top_level_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "resolved_ip": { + "type": "ip" + }, + "response_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ecs": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "error": { + "properties": { + "code": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "norms": false, + "type": "text" + }, + "stack_trace": { + "doc_values": false, + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "index": false, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "event": { + "properties": { + "action": { + "ignore_above": 1024, + "type": "keyword" + }, + "category": { + "ignore_above": 1024, + "type": "keyword" + }, + "code": { + "ignore_above": 1024, + "type": "keyword" + }, + "created": { + "type": "date" + }, + "dataset": { + "ignore_above": 1024, + "type": "keyword" + }, + "duration": { + "type": "long" + }, + "end": { + "type": "date" + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "ingested": { + "type": "date" + }, + "kind": { + "ignore_above": 1024, + "type": "keyword" + }, + "module": { + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "doc_values": false, + "ignore_above": 1024, + "index": false, + "type": "keyword" + }, + "outcome": { + "ignore_above": 1024, + "type": "keyword" + }, + "provider": { + "ignore_above": 1024, + "type": "keyword" + }, + "reason": { + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + }, + "risk_score": { + "type": "float" + }, + "risk_score_norm": { + "type": "float" + }, + "sequence": { + "type": "long" + }, + "severity": { + "type": "long" + }, + "start": { + "type": "date" + }, + "timezone": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "url": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "file": { + "properties": { + "accessed": { + "type": "date" + }, + "attributes": { + "ignore_above": 1024, + "type": "keyword" + }, + "code_signature": { + "properties": { + "exists": { + "type": "boolean" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "trusted": { + "type": "boolean" + }, + "valid": { + "type": "boolean" + } + } + }, + "created": { + "type": "date" + }, + "ctime": { + "type": "date" + }, + "device": { + "ignore_above": 1024, + "type": "keyword" + }, + "directory": { + "ignore_above": 1024, + "type": "keyword" + }, + "drive_letter": { + "ignore_above": 1, + "type": "keyword" + }, + "extension": { + "ignore_above": 1024, + "type": "keyword" + }, + "gid": { + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "ignore_above": 1024, + "type": "keyword" + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha512": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "inode": { + "ignore_above": 1024, + "type": "keyword" + }, + "mime_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "mode": { + "ignore_above": 1024, + "type": "keyword" + }, + "mtime": { + "type": "date" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "owner": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "pe": { + "properties": { + "architecture": { + "ignore_above": 1024, + "type": "keyword" + }, + "company": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "file_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "imphash": { + "ignore_above": 1024, + "type": "keyword" + }, + "original_file_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "product": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "size": { + "type": "long" + }, + "target_path": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "uid": { + "ignore_above": 1024, + "type": "keyword" + }, + "x509": { + "properties": { + "alternative_names": { + "ignore_above": 1024, + "type": "keyword" + }, + "issuer": { + "properties": { + "common_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country": { + "ignore_above": 1024, + "type": "keyword" + }, + "distinguished_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "locality": { + "ignore_above": 1024, + "type": "keyword" + }, + "organization": { + "ignore_above": 1024, + "type": "keyword" + }, + "organizational_unit": { + "ignore_above": 1024, + "type": "keyword" + }, + "state_or_province": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "not_after": { + "type": "date" + }, + "not_before": { + "type": "date" + }, + "public_key_algorithm": { + "ignore_above": 1024, + "type": "keyword" + }, + "public_key_curve": { + "ignore_above": 1024, + "type": "keyword" + }, + "public_key_exponent": { + "doc_values": false, + "index": false, + "type": "long" + }, + "public_key_size": { + "type": "long" + }, + "serial_number": { + "ignore_above": 1024, + "type": "keyword" + }, + "signature_algorithm": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject": { + "properties": { + "common_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country": { + "ignore_above": 1024, + "type": "keyword" + }, + "distinguished_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "locality": { + "ignore_above": 1024, + "type": "keyword" + }, + "organization": { + "ignore_above": 1024, + "type": "keyword" + }, + "organizational_unit": { + "ignore_above": 1024, + "type": "keyword" + }, + "state_or_province": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "version_number": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "host": { + "properties": { + "architecture": { + "ignore_above": 1024, + "type": "keyword" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "properties": { + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "uptime": { + "type": "long" + }, + "user": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "roles": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "http": { + "properties": { + "request": { + "properties": { + "body": { + "properties": { + "bytes": { + "type": "long" + }, + "content": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "bytes": { + "type": "long" + }, + "method": { + "ignore_above": 1024, + "type": "keyword" + }, + "mime_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "referrer": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "response": { + "properties": { + "body": { + "properties": { + "bytes": { + "type": "long" + }, + "content": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "bytes": { + "type": "long" + }, + "mime_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "status_code": { + "type": "long" + } + } + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "labels": { + "type": "object" + }, + "log": { + "properties": { + "file": { + "properties": { + "path": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "level": { + "ignore_above": 1024, + "type": "keyword" + }, + "logger": { + "ignore_above": 1024, + "type": "keyword" + }, + "origin": { + "properties": { + "file": { + "properties": { + "line": { + "type": "integer" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "function": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "original": { + "doc_values": false, + "ignore_above": 1024, + "index": false, + "type": "keyword" + }, + "syslog": { + "properties": { + "facility": { + "properties": { + "code": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "priority": { + "type": "long" + }, + "severity": { + "properties": { + "code": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + }, + "type": "object" + } + } + }, + "message": { + "norms": false, + "type": "text" + }, + "network": { + "properties": { + "application": { + "ignore_above": 1024, + "type": "keyword" + }, + "bytes": { + "type": "long" + }, + "community_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "direction": { + "ignore_above": 1024, + "type": "keyword" + }, + "forwarded_ip": { + "type": "ip" + }, + "iana_number": { + "ignore_above": 1024, + "type": "keyword" + }, + "inner": { + "properties": { + "vlan": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + }, + "type": "object" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "packets": { + "type": "long" + }, + "protocol": { + "ignore_above": 1024, + "type": "keyword" + }, + "transport": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "vlan": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "observer": { + "properties": { + "egress": { + "properties": { + "interface": { + "properties": { + "alias": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "vlan": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "zone": { + "ignore_above": 1024, + "type": "keyword" + } + }, + "type": "object" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "ingress": { + "properties": { + "interface": { + "properties": { + "alias": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "vlan": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "zone": { + "ignore_above": 1024, + "type": "keyword" + } + }, + "type": "object" + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "properties": { + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "product": { + "ignore_above": 1024, + "type": "keyword" + }, + "serial_number": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "vendor": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "organization": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "package": { + "properties": { + "architecture": { + "ignore_above": 1024, + "type": "keyword" + }, + "build_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "checksum": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "install_scope": { + "ignore_above": 1024, + "type": "keyword" + }, + "installed": { + "type": "date" + }, + "license": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + }, + "size": { + "type": "long" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "process": { + "properties": { + "args": { + "ignore_above": 1024, + "type": "keyword" + }, + "args_count": { + "type": "long" + }, + "code_signature": { + "properties": { + "exists": { + "type": "boolean" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "trusted": { + "type": "boolean" + }, + "valid": { + "type": "boolean" + } + } + }, + "command_line": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "entity_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "executable": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "exit_code": { + "type": "long" + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha512": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "parent": { + "properties": { + "args": { + "ignore_above": 1024, + "type": "keyword" + }, + "args_count": { + "type": "long" + }, + "code_signature": { + "properties": { + "exists": { + "type": "boolean" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "trusted": { + "type": "boolean" + }, + "valid": { + "type": "boolean" + } + } + }, + "command_line": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "entity_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "executable": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "exit_code": { + "type": "long" + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha512": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "pe": { + "properties": { + "architecture": { + "ignore_above": 1024, + "type": "keyword" + }, + "company": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "file_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "imphash": { + "ignore_above": 1024, + "type": "keyword" + }, + "original_file_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "product": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "pgid": { + "type": "long" + }, + "pid": { + "type": "long" + }, + "ppid": { + "type": "long" + }, + "start": { + "type": "date" + }, + "thread": { + "properties": { + "id": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "title": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "uptime": { + "type": "long" + }, + "working_directory": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "pe": { + "properties": { + "architecture": { + "ignore_above": 1024, + "type": "keyword" + }, + "company": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "file_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "imphash": { + "ignore_above": 1024, + "type": "keyword" + }, + "original_file_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "product": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "pgid": { + "type": "long" + }, + "pid": { + "type": "long" + }, + "ppid": { + "type": "long" + }, + "start": { + "type": "date" + }, + "thread": { + "properties": { + "id": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "title": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "uptime": { + "type": "long" + }, + "working_directory": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "registry": { + "properties": { + "data": { + "properties": { + "bytes": { + "ignore_above": 1024, + "type": "keyword" + }, + "strings": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hive": { + "ignore_above": 1024, + "type": "keyword" + }, + "key": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "value": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "related": { + "properties": { + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "hosts": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "ip" + }, + "user": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "rule": { + "properties": { + "author": { + "ignore_above": 1024, + "type": "keyword" + }, + "category": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "license": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + }, + "ruleset": { + "ignore_above": 1024, + "type": "keyword" + }, + "uuid": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "server": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "as": { + "properties": { + "number": { + "type": "long" + }, + "organization": { + "properties": { + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "bytes": { + "type": "long" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "nat": { + "properties": { + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + } + } + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + }, + "registered_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "subdomain": { + "ignore_above": 1024, + "type": "keyword" + }, + "top_level_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "roles": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "service": { + "properties": { + "ephemeral_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "node": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "source": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "as": { + "properties": { + "number": { + "type": "long" + }, + "organization": { + "properties": { + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "bytes": { + "type": "long" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "nat": { + "properties": { + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + } + } + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + }, + "registered_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "subdomain": { + "ignore_above": 1024, + "type": "keyword" + }, + "top_level_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "roles": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "span": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "tags": { + "ignore_above": 1024, + "type": "keyword" + }, + "threat": { + "properties": { + "framework": { + "ignore_above": 1024, + "type": "keyword" + }, + "tactic": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "technique": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + }, + "subtechnique": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + } + }, + "tls": { + "properties": { + "cipher": { + "ignore_above": 1024, + "type": "keyword" + }, + "client": { + "properties": { + "certificate": { + "ignore_above": 1024, + "type": "keyword" + }, + "certificate_chain": { + "ignore_above": 1024, + "type": "keyword" + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "issuer": { + "ignore_above": 1024, + "type": "keyword" + }, + "ja3": { + "ignore_above": 1024, + "type": "keyword" + }, + "not_after": { + "type": "date" + }, + "not_before": { + "type": "date" + }, + "server_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject": { + "ignore_above": 1024, + "type": "keyword" + }, + "supported_ciphers": { + "ignore_above": 1024, + "type": "keyword" + }, + "x509": { + "properties": { + "alternative_names": { + "ignore_above": 1024, + "type": "keyword" + }, + "issuer": { + "properties": { + "common_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country": { + "ignore_above": 1024, + "type": "keyword" + }, + "distinguished_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "locality": { + "ignore_above": 1024, + "type": "keyword" + }, + "organization": { + "ignore_above": 1024, + "type": "keyword" + }, + "organizational_unit": { + "ignore_above": 1024, + "type": "keyword" + }, + "state_or_province": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "not_after": { + "type": "date" + }, + "not_before": { + "type": "date" + }, + "public_key_algorithm": { + "ignore_above": 1024, + "type": "keyword" + }, + "public_key_curve": { + "ignore_above": 1024, + "type": "keyword" + }, + "public_key_exponent": { + "doc_values": false, + "index": false, + "type": "long" + }, + "public_key_size": { + "type": "long" + }, + "serial_number": { + "ignore_above": 1024, + "type": "keyword" + }, + "signature_algorithm": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject": { + "properties": { + "common_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country": { + "ignore_above": 1024, + "type": "keyword" + }, + "distinguished_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "locality": { + "ignore_above": 1024, + "type": "keyword" + }, + "organization": { + "ignore_above": 1024, + "type": "keyword" + }, + "organizational_unit": { + "ignore_above": 1024, + "type": "keyword" + }, + "state_or_province": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "version_number": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "curve": { + "ignore_above": 1024, + "type": "keyword" + }, + "established": { + "type": "boolean" + }, + "next_protocol": { + "ignore_above": 1024, + "type": "keyword" + }, + "resumed": { + "type": "boolean" + }, + "server": { + "properties": { + "certificate": { + "ignore_above": 1024, + "type": "keyword" + }, + "certificate_chain": { + "ignore_above": 1024, + "type": "keyword" + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "issuer": { + "ignore_above": 1024, + "type": "keyword" + }, + "ja3s": { + "ignore_above": 1024, + "type": "keyword" + }, + "not_after": { + "type": "date" + }, + "not_before": { + "type": "date" + }, + "subject": { + "ignore_above": 1024, + "type": "keyword" + }, + "x509": { + "properties": { + "alternative_names": { + "ignore_above": 1024, + "type": "keyword" + }, + "issuer": { + "properties": { + "common_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country": { + "ignore_above": 1024, + "type": "keyword" + }, + "distinguished_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "locality": { + "ignore_above": 1024, + "type": "keyword" + }, + "organization": { + "ignore_above": 1024, + "type": "keyword" + }, + "organizational_unit": { + "ignore_above": 1024, + "type": "keyword" + }, + "state_or_province": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "not_after": { + "type": "date" + }, + "not_before": { + "type": "date" + }, + "public_key_algorithm": { + "ignore_above": 1024, + "type": "keyword" + }, + "public_key_curve": { + "ignore_above": 1024, + "type": "keyword" + }, + "public_key_exponent": { + "doc_values": false, + "index": false, + "type": "long" + }, + "public_key_size": { + "type": "long" + }, + "serial_number": { + "ignore_above": 1024, + "type": "keyword" + }, + "signature_algorithm": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject": { + "properties": { + "common_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country": { + "ignore_above": 1024, + "type": "keyword" + }, + "distinguished_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "locality": { + "ignore_above": 1024, + "type": "keyword" + }, + "organization": { + "ignore_above": 1024, + "type": "keyword" + }, + "organizational_unit": { + "ignore_above": 1024, + "type": "keyword" + }, + "state_or_province": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "version_number": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + }, + "version_protocol": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "trace": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "transaction": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "url": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "extension": { + "ignore_above": 1024, + "type": "keyword" + }, + "fragment": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "password": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "port": { + "type": "long" + }, + "query": { + "ignore_above": 1024, + "type": "keyword" + }, + "registered_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "scheme": { + "ignore_above": 1024, + "type": "keyword" + }, + "subdomain": { + "ignore_above": 1024, + "type": "keyword" + }, + "top_level_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "username": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "user": { + "properties": { + "changes": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "roles": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "effective": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "roles": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "roles": { + "ignore_above": 1024, + "type": "keyword" + }, + "target": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "roles": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "user_agent": { + "properties": { + "device": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "properties": { + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "vulnerability": { + "properties": { + "category": { + "ignore_above": 1024, + "type": "keyword" + }, + "classification": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "enumeration": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + }, + "report_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "scanner": { + "properties": { + "vendor": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "score": { + "properties": { + "base": { + "type": "float" + }, + "environmental": { + "type": "float" + }, + "temporal": { + "type": "float" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "severity": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } +} \ No newline at end of file diff --git a/x-pack/plugins/rule_registry/server/index.ts b/x-pack/plugins/rule_registry/server/index.ts new file mode 100644 index 0000000000000..7c46717300819 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/index.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { schema, TypeOf } from '@kbn/config-schema'; +import { PluginInitializerContext } from 'src/core/server'; +import { RuleRegistryPlugin } from './plugin'; + +export { RuleRegistryPluginSetupContract } from './plugin'; +export { createLifecycleRuleTypeFactory } from './rule_registry/rule_type_helpers/create_lifecycle_rule_type_factory'; +export { ecsFieldMap } from './generated/ecs_field_map'; +export { pickWithPatterns } from './rule_registry/field_map/pick_with_patterns'; +export { FieldMapOf } from './types'; +export { ScopedRuleRegistryClient } from './rule_registry/create_scoped_rule_registry_client/types'; + +export const config = { + schema: schema.object({ + enabled: schema.boolean({ defaultValue: true }), + writeEnabled: schema.boolean({ defaultValue: false }), + }), +}; + +export type RuleRegistryConfig = TypeOf; + +export const plugin = (initContext: PluginInitializerContext) => + new RuleRegistryPlugin(initContext); diff --git a/x-pack/plugins/rule_registry/server/plugin.ts b/x-pack/plugins/rule_registry/server/plugin.ts new file mode 100644 index 0000000000000..9e83d938d508b --- /dev/null +++ b/x-pack/plugins/rule_registry/server/plugin.ts @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { PluginInitializerContext, Plugin, CoreSetup } from 'src/core/server'; +import { PluginSetupContract as AlertingPluginSetupContract } from '../../alerting/server'; +import { RuleRegistry } from './rule_registry'; +import { defaultIlmPolicy } from './rule_registry/defaults/ilm_policy'; +import { defaultFieldMap } from './rule_registry/defaults/field_map'; +import { RuleRegistryConfig } from '.'; + +export type RuleRegistryPluginSetupContract = RuleRegistry; + +export class RuleRegistryPlugin implements Plugin { + constructor(private readonly initContext: PluginInitializerContext) { + this.initContext = initContext; + } + + public setup( + core: CoreSetup, + plugins: { alerting: AlertingPluginSetupContract } + ): RuleRegistryPluginSetupContract { + const globalConfig = this.initContext.config.legacy.get(); + const config = this.initContext.config.get(); + + const logger = this.initContext.logger.get(); + + const rootRegistry = new RuleRegistry({ + coreSetup: core, + ilmPolicy: defaultIlmPolicy, + fieldMap: defaultFieldMap, + kibanaIndex: globalConfig.kibana.index, + name: 'alerts', + kibanaVersion: this.initContext.env.packageInfo.version, + logger: logger.get('root'), + alertingPluginSetupContract: plugins.alerting, + writeEnabled: config.writeEnabled, + }); + + return rootRegistry; + } + + public start() {} + + public stop() {} +} diff --git a/x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/index.ts b/x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/index.ts new file mode 100644 index 0000000000000..9a3d4a38d2ad6 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/index.ts @@ -0,0 +1,174 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { Either, isLeft, isRight } from 'fp-ts/lib/Either'; +import { Errors } from 'io-ts'; +import { PathReporter } from 'io-ts/lib/PathReporter'; +import { Logger, SavedObjectsClientContract } from 'kibana/server'; +import { IScopedClusterClient as ScopedClusterClient } from 'src/core/server'; +import { compact } from 'lodash'; +import { ESSearchRequest } from 'typings/elasticsearch'; +import { ClusterClientAdapter } from '../../../../event_log/server'; +import { runtimeTypeFromFieldMap, OutputOfFieldMap } from '../field_map/runtime_type_from_fieldmap'; +import { ScopedRuleRegistryClient, EventsOf } from './types'; +import { DefaultFieldMap } from '../defaults/field_map'; + +const getRuleUuids = async ({ + savedObjectsClient, + namespace, +}: { + savedObjectsClient: SavedObjectsClientContract; + namespace?: string; +}) => { + const options = { + type: 'alert', + ...(namespace ? { namespace } : {}), + }; + + const pitFinder = savedObjectsClient.createPointInTimeFinder({ + ...options, + }); + + const ruleUuids: string[] = []; + + for await (const response of pitFinder.find()) { + ruleUuids.push(...response.saved_objects.map((object) => object.id)); + } + + await pitFinder.close(); + + return ruleUuids; +}; + +const createPathReporterError = (either: Either) => { + const error = new Error(`Failed to validate alert event`); + error.stack += '\n' + PathReporter.report(either).join('\n'); + return error; +}; + +export function createScopedRuleRegistryClient({ + fieldMap, + scopedClusterClient, + savedObjectsClient, + namespace, + clusterClientAdapter, + indexAliasName, + indexTarget, + logger, + ruleData, +}: { + fieldMap: TFieldMap; + scopedClusterClient: ScopedClusterClient; + savedObjectsClient: SavedObjectsClientContract; + namespace?: string; + clusterClientAdapter: ClusterClientAdapter<{ + body: OutputOfFieldMap; + index: string; + }>; + indexAliasName: string; + indexTarget: string; + logger: Logger; + ruleData?: { + rule: { + id: string; + uuid: string; + category: string; + name: string; + }; + producer: string; + tags: string[]; + }; +}): ScopedRuleRegistryClient { + const docRt = runtimeTypeFromFieldMap(fieldMap); + + const defaults: Partial> = ruleData + ? { + 'rule.uuid': ruleData.rule.uuid, + 'rule.id': ruleData.rule.id, + 'rule.name': ruleData.rule.name, + 'rule.category': ruleData.rule.category, + 'kibana.rac.producer': ruleData.producer, + tags: ruleData.tags, + } + : {}; + + const client: ScopedRuleRegistryClient = { + search: async (searchRequest) => { + const ruleUuids = await getRuleUuids({ + savedObjectsClient, + namespace, + }); + + const response = await scopedClusterClient.asInternalUser.search({ + ...searchRequest, + index: indexTarget, + body: { + ...searchRequest.body, + query: { + bool: { + filter: [ + { terms: { 'rule.uuid': ruleUuids } }, + ...(searchRequest.body?.query ? [searchRequest.body.query] : []), + ], + }, + }, + }, + }); + + return { + body: response.body as any, + events: compact( + response.body.hits.hits.map((hit) => { + const validation = docRt.decode(hit.fields); + if (isLeft(validation)) { + const error = createPathReporterError(validation); + logger.error(error); + return undefined; + } + return docRt.encode(validation.right); + }) + ) as EventsOf, + }; + }, + index: (doc) => { + const validation = docRt.decode({ + ...doc, + ...defaults, + }); + + if (isLeft(validation)) { + throw createPathReporterError(validation); + } + + clusterClientAdapter.indexDocument({ body: validation.right, index: indexAliasName }); + }, + bulkIndex: (docs) => { + const validations = docs.map((doc) => { + return docRt.decode({ + ...doc, + ...defaults, + }); + }); + + const errors = compact( + validations.map((validation) => + isLeft(validation) ? createPathReporterError(validation) : null + ) + ); + + errors.forEach((error) => { + logger.error(error); + }); + + const operations = compact( + validations.map((validation) => (isRight(validation) ? validation.right : null)) + ).map((doc) => ({ body: doc, index: indexAliasName })); + + return clusterClientAdapter.indexDocuments(operations); + }, + }; + return client; +} diff --git a/x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/types.ts b/x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/types.ts new file mode 100644 index 0000000000000..95aa180709a51 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/types.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ESSearchRequest, ESSearchResponse } from 'typings/elasticsearch'; +import { DefaultFieldMap } from '../defaults/field_map'; +import { PatternsUnionOf, PickWithPatterns } from '../field_map/pick_with_patterns'; +import { OutputOfFieldMap } from '../field_map/runtime_type_from_fieldmap'; + +export type PrepopulatedRuleEventFields = + | 'rule.uuid' + | 'rule.id' + | 'rule.name' + | 'rule.type' + | 'rule.category' + | 'producer'; + +type FieldsOf = + | Array<{ field: PatternsUnionOf } | PatternsUnionOf> + | PatternsUnionOf; + +type Fields = Array<{ field: TPattern } | TPattern> | TPattern; + +type FieldsESSearchRequest = ESSearchRequest & { + body?: { fields: FieldsOf }; +}; + +export type EventsOf< + TFieldsESSearchRequest extends ESSearchRequest, + TFieldMap extends DefaultFieldMap +> = TFieldsESSearchRequest extends { body: { fields: infer TFields } } + ? TFields extends Fields + ? Array>> + : never + : never; + +export interface ScopedRuleRegistryClient { + search>( + request: TSearchRequest + ): Promise<{ + body: ESSearchResponse; + events: EventsOf; + }>; + index(doc: Omit, PrepopulatedRuleEventFields>): void; + bulkIndex( + doc: Array, PrepopulatedRuleEventFields>> + ): Promise; +} diff --git a/x-pack/plugins/rule_registry/server/rule_registry/defaults/field_map.ts b/x-pack/plugins/rule_registry/server/rule_registry/defaults/field_map.ts new file mode 100644 index 0000000000000..db851b7b94c76 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_registry/defaults/field_map.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ecsFieldMap } from '../../generated/ecs_field_map'; +import { pickWithPatterns } from '../field_map/pick_with_patterns'; + +export const defaultFieldMap = { + ...pickWithPatterns( + ecsFieldMap, + '@timestamp', + 'event.kind', + 'event.action', + 'rule.uuid', + 'rule.id', + 'rule.name', + 'rule.category', + 'tags' + ), + 'kibana.rac.producer': { type: 'keyword' }, + 'kibana.rac.alert.uuid': { type: 'keyword' }, + 'kibana.rac.alert.id': { type: 'keyword' }, + 'kibana.rac.alert.start': { type: 'date' }, + 'kibana.rac.alert.end': { type: 'date' }, + 'kibana.rac.alert.duration.us': { type: 'long' }, + 'kibana.rac.alert.severity.level': { type: 'keyword' }, + 'kibana.rac.alert.severity.value': { type: 'long' }, + 'kibana.rac.alert.status': { type: 'keyword' }, +} as const; + +export type DefaultFieldMap = typeof defaultFieldMap; diff --git a/x-pack/plugins/rule_registry/server/rule_registry/defaults/ilm_policy.ts b/x-pack/plugins/rule_registry/server/rule_registry/defaults/ilm_policy.ts new file mode 100644 index 0000000000000..c80f7e772f308 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_registry/defaults/ilm_policy.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ILMPolicy } from '../types'; + +export const defaultIlmPolicy: ILMPolicy = { + policy: { + phases: { + hot: { + actions: { + rollover: { + max_age: '90d', + max_size: '50gb', + }, + }, + }, + delete: { + actions: { + delete: {}, + }, + }, + }, + }, +}; diff --git a/x-pack/plugins/rule_registry/server/rule_registry/field_map/mapping_from_field_map.ts b/x-pack/plugins/rule_registry/server/rule_registry/field_map/mapping_from_field_map.ts new file mode 100644 index 0000000000000..6e4e13b01d2c5 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_registry/field_map/mapping_from_field_map.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { set } from '@elastic/safer-lodash-set'; +import { FieldMap, Mappings } from '../types'; + +export function mappingFromFieldMap(fieldMap: FieldMap): Mappings { + const mappings = { + dynamic: 'strict' as const, + properties: {}, + }; + + const fields = Object.keys(fieldMap).map((key) => { + const field = fieldMap[key]; + return { + name: key, + ...field, + }; + }); + + fields.forEach((field) => { + const { name, required, array, ...rest } = field; + + set(mappings.properties, field.name.split('.').join('.properties.'), rest); + }); + + return mappings; +} diff --git a/x-pack/plugins/rule_registry/server/rule_registry/field_map/merge_field_maps.ts b/x-pack/plugins/rule_registry/server/rule_registry/field_map/merge_field_maps.ts new file mode 100644 index 0000000000000..e15b228b0f287 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_registry/field_map/merge_field_maps.ts @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import util from 'util'; +import { FieldMap } from '../types'; + +export function mergeFieldMaps( + first: T1, + second: T2 +): T1 & T2 { + const conflicts: Array> = []; + + Object.keys(second).forEach((name) => { + const field = second[name]; + + const parts = name.split('.'); + + const parents = parts.slice(0, parts.length - 2).map((part, index, array) => { + return [...array.slice(0, index - 1), part].join('.'); + }); + + parents + .filter((parent) => first[parent] !== undefined) + .forEach((parent) => { + conflicts.push({ + [parent]: [{ type: 'object' }, first[parent]!], + }); + }); + + if (first[name]) { + conflicts.push({ + [name]: [field, first[name]], + }); + } + }); + + if (conflicts.length) { + const err = new Error(`Could not merge mapping due to conflicts`); + Object.assign(err, { conflicts: util.inspect(conflicts, { depth: null }) }); + throw err; + } + + return { + ...first, + ...second, + }; +} diff --git a/x-pack/plugins/rule_registry/server/rule_registry/field_map/pick_with_patterns.test.ts b/x-pack/plugins/rule_registry/server/rule_registry/field_map/pick_with_patterns.test.ts new file mode 100644 index 0000000000000..48ba7c873db25 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_registry/field_map/pick_with_patterns.test.ts @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { pickWithPatterns } from './pick_with_patterns'; + +describe('pickWithPatterns', () => { + const fieldMap = { + 'event.category': { type: 'keyword' }, + 'event.kind': { type: 'keyword' }, + 'destination.bytes': { + type: 'long', + array: false, + required: false, + }, + 'destination.domain': { + type: 'keyword', + array: false, + required: false, + }, + 'destination.geo.city_name': { + type: 'keyword', + array: false, + required: false, + }, + } as const; + + it('picks a single field', () => { + expect(Object.keys(pickWithPatterns(fieldMap, 'event.category'))).toEqual(['event.category']); + }); + + it('picks event fields', () => { + expect(Object.keys(pickWithPatterns(fieldMap, 'event.*')).sort()).toEqual([ + 'event.category', + 'event.kind', + ]); + }); + + it('picks destination.geo fields', () => { + expect(Object.keys(pickWithPatterns(fieldMap, 'destination.geo.*')).sort()).toEqual([ + 'destination.geo.city_name', + ]); + }); + + it('picks all destination fields', () => { + expect(Object.keys(pickWithPatterns(fieldMap, 'destination.*')).sort()).toEqual([ + 'destination.bytes', + 'destination.domain', + 'destination.geo.city_name', + ]); + }); + + it('picks fields from multiple patterns', () => { + expect( + Object.keys(pickWithPatterns(fieldMap, 'destination.geo.*', 'event.category')).sort() + ).toEqual(['destination.geo.city_name', 'event.category']); + }); + + it('picks all fields', () => { + expect(Object.keys(pickWithPatterns(fieldMap, '*')).sort()).toEqual([ + 'destination.bytes', + 'destination.domain', + 'destination.geo.city_name', + 'event.category', + 'event.kind', + ]); + }); +}); diff --git a/x-pack/plugins/rule_registry/server/rule_registry/field_map/pick_with_patterns.ts b/x-pack/plugins/rule_registry/server/rule_registry/field_map/pick_with_patterns.ts new file mode 100644 index 0000000000000..f8a88957fceb5 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_registry/field_map/pick_with_patterns.ts @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ValuesType, SetIntersection, OmitByValueExact } from 'utility-types'; +import { pick } from 'lodash'; + +type SplitByDot< + TPath extends string, + TPrefix extends string = '' +> = TPath extends `${infer TKey}.${infer TRest}` + ? [`${TPrefix}${TKey}.*`, ...SplitByDot] + : [`${TPrefix}${TPath}`]; + +type PatternMapOf> = { + [TKey in keyof T]: ValuesType] : never>; +}; + +export type PickWithPatterns< + T extends Record, + TPatterns extends string[] +> = OmitByValueExact< + { + [TFieldName in keyof T]: SetIntersection< + ValuesType, + PatternMapOf[TFieldName] + > extends never + ? never + : T[TFieldName]; + }, + never +>; + +export type PatternsUnionOf> = '*' | ValuesType>; + +export function pickWithPatterns< + T extends Record, + TPatterns extends Array> +>(map: T, ...patterns: TPatterns): PickWithPatterns { + const allFields = Object.keys(map); + const matchedFields = allFields.filter((field) => + patterns.some((pattern) => { + if (pattern === field) { + return true; + } + + const fieldParts = field.split('.'); + const patternParts = pattern.split('.'); + + if (patternParts.indexOf('*') !== patternParts.length - 1) { + return false; + } + + return fieldParts.every((fieldPart, index) => { + const patternPart = patternParts.length - 1 < index ? '*' : patternParts[index]; + + return fieldPart === patternPart || patternPart === '*'; + }); + }) + ); + + return (pick(map, matchedFields) as unknown) as PickWithPatterns; +} diff --git a/x-pack/plugins/rule_registry/server/rule_registry/field_map/runtime_type_from_fieldmap.test.ts b/x-pack/plugins/rule_registry/server/rule_registry/field_map/runtime_type_from_fieldmap.test.ts new file mode 100644 index 0000000000000..0acf80bfb42e5 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_registry/field_map/runtime_type_from_fieldmap.test.ts @@ -0,0 +1,95 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { runtimeTypeFromFieldMap } from './runtime_type_from_fieldmap'; + +describe('runtimeTypeFromFieldMap', () => { + const fieldmapRt = runtimeTypeFromFieldMap({ + keywordField: { type: 'keyword' }, + longField: { type: 'long' }, + requiredKeywordField: { type: 'keyword', required: true }, + multiKeywordField: { type: 'keyword', array: true }, + } as const); + + it('accepts both singular and array fields', () => { + expect( + fieldmapRt.is({ + requiredKeywordField: 'keyword', + }) + ).toBe(true); + + expect( + fieldmapRt.is({ + requiredKeywordField: ['keyword'], + }) + ).toBe(true); + + expect( + fieldmapRt.is({ + requiredKeywordField: ['keyword'], + multiKeywordField: 'keyword', + }) + ).toBe(true); + + expect( + fieldmapRt.is({ + requiredKeywordField: ['keyword'], + multiKeywordField: ['keyword'], + }) + ).toBe(true); + }); + + it('fails on invalid data types', () => { + expect( + fieldmapRt.is({ + requiredKeywordField: 2, + }) + ).toBe(false); + + expect( + fieldmapRt.is({ + requiredKeywordField: [2], + }) + ).toBe(false); + + expect( + fieldmapRt.is({ + requiredKeywordField: ['keyword'], + longField: ['keyword'], + }) + ).toBe(false); + + expect( + fieldmapRt.is({ + requiredKeywordField: ['keyword'], + longField: [3], + }) + ).toBe(true); + + expect( + fieldmapRt.is({ + requiredKeywordField: ['keyword'], + longField: 3, + }) + ).toBe(true); + }); + + it('outputs to single or array values', () => { + expect( + fieldmapRt.encode({ + requiredKeywordField: ['required'], + keywordField: 'keyword', + longField: [3, 2], + multiKeywordField: ['keyword', 'foo'], + }) + ).toEqual({ + requiredKeywordField: 'required', + keywordField: 'keyword', + longField: 3, + multiKeywordField: ['keyword', 'foo'], + }); + }); +}); diff --git a/x-pack/plugins/rule_registry/server/rule_registry/field_map/runtime_type_from_fieldmap.ts b/x-pack/plugins/rule_registry/server/rule_registry/field_map/runtime_type_from_fieldmap.ts new file mode 100644 index 0000000000000..6dc557c016d1a --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_registry/field_map/runtime_type_from_fieldmap.ts @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { mapValues, pickBy } from 'lodash'; +import * as t from 'io-ts'; +import { Mutable, PickByValueExact } from 'utility-types'; +import { FieldMap } from '../types'; + +const esFieldTypeMap = { + keyword: t.string, + text: t.string, + date: t.string, + boolean: t.boolean, + byte: t.number, + long: t.number, + integer: t.number, + short: t.number, + double: t.number, + float: t.number, + scaled_float: t.number, + unsigned_long: t.number, + flattened: t.record(t.string, t.array(t.string)), +}; + +type EsFieldTypeMap = typeof esFieldTypeMap; + +type EsFieldTypeOf = T extends keyof EsFieldTypeMap + ? EsFieldTypeMap[T] + : t.UnknownC; + +type RequiredKeysOf> = keyof PickByValueExact< + { + [key in keyof T]: T[key]['required']; + }, + true +>; + +type IntersectionTypeOf< + T extends Record +> = t.IntersectionC< + [ + t.TypeC>>, + t.PartialC<{ [key in keyof T]: T[key]['type'] }> + ] +>; + +type CastArray> = t.Type< + t.TypeOf | Array>, + Array>, + unknown +>; +type CastSingle> = t.Type< + t.TypeOf | Array>, + t.TypeOf, + unknown +>; + +const createCastArrayRt = >(type: T): CastArray => { + const union = t.union([type, t.array(type)]); + + return new t.Type('castArray', union.is, union.validate, (a) => (Array.isArray(a) ? a : [a])); +}; + +const createCastSingleRt = >(type: T): CastSingle => { + const union = t.union([type, t.array(type)]); + + return new t.Type('castSingle', union.is, union.validate, (a) => (Array.isArray(a) ? a[0] : a)); +}; + +type MapTypeValues = { + [key in keyof T]: { + required: T[key]['required']; + type: T[key]['array'] extends true + ? CastArray> + : CastSingle>; + }; +}; + +type FieldMapType = IntersectionTypeOf>; + +export type TypeOfFieldMap = Mutable>>; +export type OutputOfFieldMap = Mutable>>; + +export function runtimeTypeFromFieldMap( + fieldMap: TFieldMap +): FieldMapType { + function mapToType(fields: FieldMap) { + return mapValues(fields, (field, key) => { + const type = + field.type in esFieldTypeMap + ? esFieldTypeMap[field.type as keyof EsFieldTypeMap] + : t.unknown; + + return field.array ? createCastArrayRt(type) : createCastSingleRt(type); + }); + } + + const required = pickBy(fieldMap, (field) => field.required); + + return (t.intersection([ + t.exact(t.partial(mapToType(fieldMap))), + t.type(mapToType(required)), + ]) as unknown) as FieldMapType; +} diff --git a/x-pack/plugins/rule_registry/server/rule_registry/index.ts b/x-pack/plugins/rule_registry/server/rule_registry/index.ts new file mode 100644 index 0000000000000..f1d24550ade0a --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_registry/index.ts @@ -0,0 +1,240 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { CoreSetup, Logger, RequestHandlerContext } from 'kibana/server'; +import { inspect } from 'util'; +import { SpacesServiceStart } from '../../../spaces/server'; +import { + ActionVariable, + AlertInstanceState, + AlertTypeParams, + AlertTypeState, +} from '../../../alerting/common'; +import { createReadySignal, ClusterClientAdapter } from '../../../event_log/server'; +import { FieldMap, ILMPolicy } from './types'; +import { RuleParams, RuleType } from '../types'; +import { mergeFieldMaps } from './field_map/merge_field_maps'; +import { OutputOfFieldMap } from './field_map/runtime_type_from_fieldmap'; +import { mappingFromFieldMap } from './field_map/mapping_from_field_map'; +import { PluginSetupContract as AlertingPluginSetupContract } from '../../../alerting/server'; +import { createScopedRuleRegistryClient } from './create_scoped_rule_registry_client'; +import { DefaultFieldMap } from './defaults/field_map'; +import { ScopedRuleRegistryClient } from './create_scoped_rule_registry_client/types'; + +interface RuleRegistryOptions { + kibanaIndex: string; + kibanaVersion: string; + name: string; + logger: Logger; + coreSetup: CoreSetup; + spacesStart?: SpacesServiceStart; + fieldMap: TFieldMap; + ilmPolicy: ILMPolicy; + alertingPluginSetupContract: AlertingPluginSetupContract; + writeEnabled: boolean; +} + +export class RuleRegistry { + private readonly esAdapter: ClusterClientAdapter<{ + body: OutputOfFieldMap; + index: string; + }>; + private readonly children: Array> = []; + + constructor(private readonly options: RuleRegistryOptions) { + const { logger, coreSetup } = options; + + const { wait, signal } = createReadySignal(); + + this.esAdapter = new ClusterClientAdapter<{ + body: OutputOfFieldMap; + index: string; + }>({ + wait, + elasticsearchClientPromise: coreSetup + .getStartServices() + .then(([{ elasticsearch }]) => elasticsearch.client.asInternalUser), + logger: logger.get('esAdapter'), + }); + + if (this.options.writeEnabled) { + this.initialize() + .then(() => { + this.options.logger.debug('Bootstrapped alerts index'); + signal(true); + }) + .catch((err) => { + logger.error(inspect(err, { depth: null })); + signal(false); + }); + } else { + logger.debug('Write disabled, indices are not being bootstrapped'); + } + } + + private getEsNames() { + const base = [this.options.kibanaIndex, this.options.name]; + const indexTarget = `${base.join('-')}*`; + const indexAliasName = [...base, this.options.kibanaVersion.toLowerCase()].join('-'); + const policyName = [...base, 'policy'].join('-'); + + return { + indexAliasName, + indexTarget, + policyName, + }; + } + + private async initialize() { + const { indexAliasName, policyName } = this.getEsNames(); + + const ilmPolicyExists = await this.esAdapter.doesIlmPolicyExist(policyName); + + if (!ilmPolicyExists) { + await this.esAdapter.createIlmPolicy( + policyName, + (this.options.ilmPolicy as unknown) as Record + ); + } + + const templateExists = await this.esAdapter.doesIndexTemplateExist(indexAliasName); + + if (!templateExists) { + await this.esAdapter.createIndexTemplate(indexAliasName, { + index_patterns: [`${indexAliasName}-*`], + settings: { + number_of_shards: 1, + auto_expand_replicas: '0-1', + 'index.lifecycle.name': policyName, + 'index.lifecycle.rollover_alias': indexAliasName, + 'sort.field': '@timestamp', + 'sort.order': 'desc', + }, + mappings: mappingFromFieldMap(this.options.fieldMap), + }); + } + + const aliasExists = await this.esAdapter.doesAliasExist(indexAliasName); + + if (!aliasExists) { + await this.esAdapter.createIndex(`${indexAliasName}-000001`, { + aliases: { + [indexAliasName]: { + is_write_index: true, + }, + }, + }); + } + } + + createScopedRuleRegistryClient({ + context, + }: { + context: RequestHandlerContext; + }): ScopedRuleRegistryClient | undefined { + if (!this.options.writeEnabled) { + return undefined; + } + const { indexAliasName, indexTarget } = this.getEsNames(); + + return createScopedRuleRegistryClient({ + savedObjectsClient: context.core.savedObjects.getClient({ includedHiddenTypes: ['alert'] }), + scopedClusterClient: context.core.elasticsearch.client, + clusterClientAdapter: this.esAdapter, + fieldMap: this.options.fieldMap, + indexAliasName, + indexTarget, + logger: this.options.logger, + }); + } + + registerType( + type: RuleType + ) { + const logger = this.options.logger.get(type.id); + + const { indexAliasName, indexTarget } = this.getEsNames(); + + this.options.alertingPluginSetupContract.registerType< + AlertTypeParams, + AlertTypeState, + AlertInstanceState, + { [key in TActionVariable['name']]: any }, + string + >({ + ...type, + executor: async (executorOptions) => { + const { services, namespace, alertId, name, tags } = executorOptions; + + const rule = { + id: type.id, + uuid: alertId, + category: type.name, + name, + }; + + const producer = type.producer; + + return type.executor({ + ...executorOptions, + rule, + producer, + services: { + ...services, + logger, + ...(this.options.writeEnabled + ? { + scopedRuleRegistryClient: createScopedRuleRegistryClient({ + savedObjectsClient: services.savedObjectsClient, + scopedClusterClient: services.scopedClusterClient, + clusterClientAdapter: this.esAdapter, + fieldMap: this.options.fieldMap, + indexAliasName, + indexTarget, + namespace, + ruleData: { + producer, + rule, + tags, + }, + logger: this.options.logger, + }), + } + : {}), + }, + }); + }, + }); + } + + create({ + name, + fieldMap, + ilmPolicy, + }: { + name: string; + fieldMap: TNextFieldMap; + ilmPolicy?: ILMPolicy; + }): RuleRegistry { + const mergedFieldMap = fieldMap + ? mergeFieldMaps(this.options.fieldMap, fieldMap) + : this.options.fieldMap; + + const child = new RuleRegistry({ + ...this.options, + logger: this.options.logger.get(name), + name: [this.options.name, name].filter(Boolean).join('-'), + fieldMap: mergedFieldMap, + ...(ilmPolicy ? { ilmPolicy } : {}), + }); + + this.children.push(child); + + // @ts-expect-error could be instantiated with a different subtype of constraint + return child; + } +} diff --git a/x-pack/plugins/rule_registry/server/rule_registry/rule_type_helpers/create_lifecycle_rule_type_factory.ts b/x-pack/plugins/rule_registry/server/rule_registry/rule_type_helpers/create_lifecycle_rule_type_factory.ts new file mode 100644 index 0000000000000..9c64e85f839bb --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_registry/rule_type_helpers/create_lifecycle_rule_type_factory.ts @@ -0,0 +1,235 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import * as t from 'io-ts'; +import { isLeft } from 'fp-ts/lib/Either'; +import v4 from 'uuid/v4'; +import { AlertInstance } from '../../../../alerting/server'; +import { ActionVariable, AlertInstanceState } from '../../../../alerting/common'; +import { RuleParams, RuleType } from '../../types'; +import { DefaultFieldMap } from '../defaults/field_map'; +import { OutputOfFieldMap } from '../field_map/runtime_type_from_fieldmap'; +import { PrepopulatedRuleEventFields } from '../create_scoped_rule_registry_client/types'; +import { RuleRegistry } from '..'; + +type UserDefinedAlertFields = Omit< + OutputOfFieldMap, + PrepopulatedRuleEventFields | 'kibana.rac.alert.id' | 'kibana.rac.alert.uuid' | '@timestamp' +>; + +type LifecycleAlertService< + TFieldMap extends DefaultFieldMap, + TActionVariable extends ActionVariable +> = (alert: { + id: string; + fields: UserDefinedAlertFields; +}) => AlertInstance; + +type CreateLifecycleRuleType = < + TRuleParams extends RuleParams, + TActionVariable extends ActionVariable +>( + type: RuleType< + TFieldMap, + TRuleParams, + TActionVariable, + { alertWithLifecycle: LifecycleAlertService } + > +) => RuleType; + +const trackedAlertStateRt = t.type({ + alertId: t.string, + alertUuid: t.string, + started: t.string, +}); + +const wrappedStateRt = t.type({ + wrapped: t.record(t.string, t.unknown), + trackedAlerts: t.record(t.string, trackedAlertStateRt), +}); + +export function createLifecycleRuleTypeFactory< + TRuleRegistry extends RuleRegistry +>(): TRuleRegistry extends RuleRegistry + ? CreateLifecycleRuleType + : never; + +export function createLifecycleRuleTypeFactory(): CreateLifecycleRuleType { + return (type) => { + return { + ...type, + executor: async (options) => { + const { + services: { scopedRuleRegistryClient, alertInstanceFactory, logger }, + state: previousState, + rule, + } = options; + + const decodedState = wrappedStateRt.decode(previousState); + + const state = isLeft(decodedState) + ? { + wrapped: previousState, + trackedAlerts: {}, + } + : decodedState.right; + + const currentAlerts: Record< + string, + UserDefinedAlertFields & { 'kibana.rac.alert.id': string } + > = {}; + + const timestamp = options.startedAt.toISOString(); + + const nextWrappedState = await type.executor({ + ...options, + state: state.wrapped, + services: { + ...options.services, + alertWithLifecycle: ({ id, fields }) => { + currentAlerts[id] = { + ...fields, + 'kibana.rac.alert.id': id, + }; + return alertInstanceFactory(id); + }, + }, + }); + + const currentAlertIds = Object.keys(currentAlerts); + const trackedAlertIds = Object.keys(state.trackedAlerts); + const newAlertIds = currentAlertIds.filter((alertId) => !trackedAlertIds.includes(alertId)); + + const allAlertIds = [...new Set(currentAlertIds.concat(trackedAlertIds))]; + + const trackedAlertStatesOfRecovered = Object.values(state.trackedAlerts).filter( + (trackedAlertState) => !currentAlerts[trackedAlertState.alertId] + ); + + logger.debug( + `Tracking ${allAlertIds.length} alerts (${newAlertIds.length} new, ${trackedAlertStatesOfRecovered.length} recovered)` + ); + + const alertsDataMap: Record> = { + ...currentAlerts, + }; + + if (scopedRuleRegistryClient && trackedAlertStatesOfRecovered.length) { + const { events } = await scopedRuleRegistryClient.search({ + body: { + query: { + bool: { + filter: [ + { + term: { + 'rule.uuid': rule.uuid, + }, + }, + { + terms: { + 'kibana.rac.alert.uuid': trackedAlertStatesOfRecovered.map( + (trackedAlertState) => trackedAlertState.alertUuid + ), + }, + }, + ], + }, + }, + size: trackedAlertStatesOfRecovered.length, + collapse: { + field: 'kibana.rac.alert.uuid', + }, + _source: false, + fields: ['*'], + sort: { + '@timestamp': 'desc' as const, + }, + }, + }); + + events.forEach((event) => { + const alertId = event['kibana.rac.alert.id']!; + alertsDataMap[alertId] = event; + }); + } + + const eventsToIndex: Array> = allAlertIds.map( + (alertId) => { + const alertData = alertsDataMap[alertId]; + + if (!alertData) { + logger.warn(`Could not find alert data for ${alertId}`); + } + + const event: OutputOfFieldMap = { + ...alertData, + '@timestamp': timestamp, + 'event.kind': 'state', + 'kibana.rac.alert.id': alertId, + }; + + const isNew = !state.trackedAlerts[alertId]; + const isRecovered = !currentAlerts[alertId]; + const isActiveButNotNew = !isNew && !isRecovered; + const isActive = !isRecovered; + + const { alertUuid, started } = state.trackedAlerts[alertId] ?? { + alertUuid: v4(), + started: timestamp, + }; + + event['kibana.rac.alert.start'] = started; + event['kibana.rac.alert.uuid'] = alertUuid; + + if (isNew) { + event['event.action'] = 'open'; + } + + if (isRecovered) { + event['kibana.rac.alert.end'] = timestamp; + event['event.action'] = 'close'; + event['kibana.rac.alert.status'] = 'closed'; + } + + if (isActiveButNotNew) { + event['event.action'] = 'active'; + } + + if (isActive) { + event['kibana.rac.alert.status'] = 'open'; + } + + event['kibana.rac.alert.duration.us'] = + (options.startedAt.getTime() - new Date(event['kibana.rac.alert.start']!).getTime()) * + 1000; + + return event; + } + ); + + if (eventsToIndex.length && scopedRuleRegistryClient) { + await scopedRuleRegistryClient.bulkIndex(eventsToIndex); + } + + const nextTrackedAlerts = Object.fromEntries( + eventsToIndex + .filter((event) => event['kibana.rac.alert.status'] !== 'closed') + .map((event) => { + const alertId = event['kibana.rac.alert.id']!; + const alertUuid = event['kibana.rac.alert.uuid']!; + const started = new Date(event['kibana.rac.alert.start']!).toISOString(); + return [alertId, { alertId, alertUuid, started }]; + }) + ); + + return { + wrapped: nextWrappedState, + trackedAlerts: nextTrackedAlerts, + }; + }, + }; + }; +} diff --git a/x-pack/plugins/rule_registry/server/rule_registry/types.ts b/x-pack/plugins/rule_registry/server/rule_registry/types.ts new file mode 100644 index 0000000000000..f6baf8bcecbd0 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_registry/types.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export interface Mappings { + dynamic: 'strict' | boolean; + properties: Record; +} + +enum ILMPolicyPhase { + hot = 'hot', + delete = 'delete', +} + +enum ILMPolicyAction { + rollover = 'rollover', + delete = 'delete', +} + +interface ILMActionOptions { + [ILMPolicyAction.rollover]: { + max_size: string; + max_age: string; + }; + [ILMPolicyAction.delete]: {}; +} + +export interface ILMPolicy { + policy: { + phases: Record< + ILMPolicyPhase, + { + actions: { + [key in keyof ILMActionOptions]?: ILMActionOptions[key]; + }; + } + >; + }; +} + +export type FieldMap = Record; diff --git a/x-pack/plugins/rule_registry/server/types.ts b/x-pack/plugins/rule_registry/server/types.ts new file mode 100644 index 0000000000000..e6b53a8558964 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/types.ts @@ -0,0 +1,100 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { Type, TypeOf } from '@kbn/config-schema'; +import { Logger } from 'kibana/server'; +import { + ActionVariable, + AlertInstanceContext, + AlertInstanceState, + AlertTypeParams, + AlertTypeState, +} from '../../alerting/common'; +import { ActionGroup, AlertExecutorOptions } from '../../alerting/server'; +import { RuleRegistry } from './rule_registry'; +import { ScopedRuleRegistryClient } from './rule_registry/create_scoped_rule_registry_client/types'; +import { DefaultFieldMap } from './rule_registry/defaults/field_map'; + +export type RuleParams = Type; + +type TypeOfRuleParams = TypeOf; + +type RuleExecutorServices< + TFieldMap extends DefaultFieldMap, + TActionVariable extends ActionVariable +> = AlertExecutorOptions< + AlertTypeParams, + AlertTypeState, + AlertInstanceState, + { [key in TActionVariable['name']]: any }, + string +>['services'] & { + logger: Logger; + scopedRuleRegistryClient?: ScopedRuleRegistryClient; +}; + +type PassthroughAlertExecutorOptions = Pick< + AlertExecutorOptions< + AlertTypeParams, + AlertTypeState, + AlertInstanceState, + AlertInstanceContext, + string + >, + 'previousStartedAt' | 'startedAt' | 'state' +>; + +type RuleExecutorFunction< + TFieldMap extends DefaultFieldMap, + TRuleParams extends RuleParams, + TActionVariable extends ActionVariable, + TAdditionalRuleExecutorServices extends Record +> = ( + options: PassthroughAlertExecutorOptions & { + services: RuleExecutorServices & TAdditionalRuleExecutorServices; + params: TypeOfRuleParams; + rule: { + id: string; + uuid: string; + name: string; + category: string; + }; + producer: string; + } +) => Promise>; + +interface RuleTypeBase { + id: string; + name: string; + actionGroups: Array>; + defaultActionGroupId: string; + producer: string; + minimumLicenseRequired: 'basic' | 'gold' | 'trial'; +} + +export type RuleType< + TFieldMap extends DefaultFieldMap, + TRuleParams extends RuleParams, + TActionVariable extends ActionVariable, + TAdditionalRuleExecutorServices extends Record = {} +> = RuleTypeBase & { + validate: { + params: TRuleParams; + }; + actionVariables: { + context: TActionVariable[]; + }; + executor: RuleExecutorFunction< + TFieldMap, + TRuleParams, + TActionVariable, + TAdditionalRuleExecutorServices + >; +}; + +export type FieldMapOf< + TRuleRegistry extends RuleRegistry +> = TRuleRegistry extends RuleRegistry ? TFieldMap : never; diff --git a/x-pack/plugins/rule_registry/tsconfig.json b/x-pack/plugins/rule_registry/tsconfig.json new file mode 100644 index 0000000000000..2961abe6cfecd --- /dev/null +++ b/x-pack/plugins/rule_registry/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "composite": true, + "outDir": "./target/types", + "emitDeclarationOnly": true, + "declaration": true, + "declarationMap": true + }, + "include": ["common/**/*", "server/**/*", "../../../typings/**/*"], + "references": [ + { "path": "../../../src/core/tsconfig.json" }, + { "path": "../alerting/tsconfig.json" }, + ] +} diff --git a/x-pack/scripts/functional_tests.js b/x-pack/scripts/functional_tests.js index 6321aa8880587..b7df493a1036a 100644 --- a/x-pack/scripts/functional_tests.js +++ b/x-pack/scripts/functional_tests.js @@ -34,6 +34,7 @@ const onlyNotInCoverageTests = [ require.resolve('../test/case_api_integration/basic/config.ts'), require.resolve('../test/apm_api_integration/basic/config.ts'), require.resolve('../test/apm_api_integration/trial/config.ts'), + require.resolve('../test/apm_api_integration/rules/config.ts'), require.resolve('../test/detection_engine_api_integration/security_and_spaces/config.ts'), require.resolve('../test/detection_engine_api_integration/basic/config.ts'), require.resolve('../test/lists_api_integration/security_and_spaces/config.ts'), diff --git a/x-pack/test/apm_api_integration/common/config.ts b/x-pack/test/apm_api_integration/common/config.ts index 04ce83323ee66..732f14d2a7bc8 100644 --- a/x-pack/test/apm_api_integration/common/config.ts +++ b/x-pack/test/apm_api_integration/common/config.ts @@ -18,6 +18,7 @@ import { registry } from './registry'; interface Config { name: APMFtrConfigName; license: 'basic' | 'trial'; + kibanaConfig?: Record; } const supertestAsApmUser = (kibanaServer: UrlObject, apmUser: ApmUser) => async ( @@ -37,7 +38,7 @@ const supertestAsApmUser = (kibanaServer: UrlObject, apmUser: ApmUser) => async }; export function createTestConfig(config: Config) { - const { license, name } = config; + const { license, name, kibanaConfig } = config; return async ({ readConfigFile }: FtrConfigProviderContext) => { const xPackAPITestsConfig = await readConfigFile( @@ -79,7 +80,15 @@ export function createTestConfig(config: Config) { ...xPackAPITestsConfig.get('esTestCluster'), license, }, - kbnTestServer: xPackAPITestsConfig.get('kbnTestServer'), + kbnTestServer: { + ...xPackAPITestsConfig.get('kbnTestServer'), + serverArgs: [ + ...xPackAPITestsConfig.get('kbnTestServer.serverArgs'), + ...(kibanaConfig + ? Object.entries(kibanaConfig).map(([key, value]) => `--${key}=${value}`) + : []), + ], + }, }; }; } diff --git a/x-pack/test/apm_api_integration/configs/index.ts b/x-pack/test/apm_api_integration/configs/index.ts index 4eeb57e3c86c4..91437a2d22e27 100644 --- a/x-pack/test/apm_api_integration/configs/index.ts +++ b/x-pack/test/apm_api_integration/configs/index.ts @@ -15,6 +15,12 @@ const apmFtrConfigs = { trial: { license: 'trial' as const, }, + rules: { + license: 'trial' as const, + kibanaConfig: { + 'xpack.ruleRegistry.writeEnabled': 'true', + }, + }, }; export type APMFtrConfigName = keyof typeof apmFtrConfigs; diff --git a/x-pack/test/apm_api_integration/rules/config.ts b/x-pack/test/apm_api_integration/rules/config.ts new file mode 100644 index 0000000000000..9830d516eb80a --- /dev/null +++ b/x-pack/test/apm_api_integration/rules/config.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { configs } from '../configs'; + +export default configs.rules; diff --git a/x-pack/test/apm_api_integration/tests/alerts/rule_registry.ts b/x-pack/test/apm_api_integration/tests/alerts/rule_registry.ts new file mode 100644 index 0000000000000..97026d126d2a1 --- /dev/null +++ b/x-pack/test/apm_api_integration/tests/alerts/rule_registry.ts @@ -0,0 +1,387 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { get, merge, omit } from 'lodash'; +import { FtrProviderContext } from '../../common/ftr_provider_context'; +import { registry } from '../../common/registry'; + +interface Alert { + schedule: { + interval: string; + }; + updatedAt: string; + executionStatus: { + lastExecutionDate: string; + status: string; + }; + updatedBy: string; + id: string; + params: Record; + scheduledTaskId: string; +} + +export default function ApiTest({ getService }: FtrProviderContext) { + const supertest = getService('supertestAsApmWriteUser'); + const es = getService('es'); + + const MAX_POLLS = 5; + const BULK_INDEX_DELAY = 1000; + const INDEXING_DELAY = 5000; + + const ALERTS_INDEX_TARGET = '.kibana-alerts-*-apm*'; + const APM_TRANSACTION_INDEX_NAME = 'apm-8.0.0-transaction'; + + const createTransactionEvent = (override: Record) => { + const now = Date.now(); + + const time = now - INDEXING_DELAY; + + return merge( + { + '@timestamp': new Date(time).toISOString(), + service: { + name: 'opbeans-go', + }, + event: { + outcome: 'success', + }, + transaction: { + duration: { + us: 1000000, + }, + type: 'request', + }, + processor: { + event: 'transaction', + }, + observer: { + version_major: 7, + }, + }, + override + ); + }; + + async function waitUntilNextExecution( + alert: Alert, + intervalInSeconds: number = 1, + count: number = 0 + ): Promise { + await new Promise((resolve) => { + setTimeout(resolve, intervalInSeconds * 1000); + }); + + const { body, status } = await supertest + .get(`/api/alerts/alert/${alert.id}`) + .set('kbn-xsrf', 'foo'); + + if (status >= 300) { + const error = new Error('Error getting alert'); + Object.assign(error, { response: { body, status } }); + throw error; + } + + const nextAlert = body as Alert; + + if (nextAlert.executionStatus.lastExecutionDate !== alert.executionStatus.lastExecutionDate) { + await new Promise((resolve) => { + setTimeout(resolve, BULK_INDEX_DELAY); + }); + await es.indices.refresh({ + index: ALERTS_INDEX_TARGET, + }); + + return nextAlert; + } + + if (count >= MAX_POLLS) { + throw new Error('Maximum number of polls exceeded'); + } + + return waitUntilNextExecution(alert, intervalInSeconds, count + 1); + } + + registry.when('Rule registry with write enabled', { config: 'rules', archives: [] }, () => { + it('bootstraps the apm alert indices', async () => { + const { body } = await es.indices.get({ + index: ALERTS_INDEX_TARGET, + expand_wildcards: 'open', + allow_no_indices: false, + }); + + const indices = Object.entries(body).map(([indexName, index]) => { + return { + indexName, + index, + }; + }); + + const indexNames = indices.map((index) => index.indexName); + + const apmIndex = indices[0]; + + // make sure it only creates one index + expect(indices.length).to.be(1); + + const apmIndexName = apmIndex.indexName; + + expect(apmIndexName.split('-').includes('observability')).to.be(true); + expect(apmIndexName.split('-').includes('apm')).to.be(true); + + expect(indexNames[0].startsWith('.kibana-alerts-observability-apm')).to.be(true); + + expect(get(apmIndex, 'index.mappings.properties.service.properties.environment.type')).to.be( + 'keyword' + ); + }); + + describe('when creating a rule', () => { + let createResponse: { + alert: Alert; + status: number; + }; + + before(async () => { + await es.indices.create({ + index: APM_TRANSACTION_INDEX_NAME, + body: { + mappings: { + dynamic: 'strict', + properties: { + event: { + properties: { + outcome: { + type: 'keyword', + }, + }, + }, + processor: { + properties: { + event: { + type: 'keyword', + }, + }, + }, + observer: { + properties: { + version_major: { + type: 'byte', + }, + }, + }, + service: { + properties: { + name: { + type: 'keyword', + }, + environment: { + type: 'keyword', + }, + }, + }, + transaction: { + properties: { + type: { + type: 'keyword', + }, + duration: { + properties: { + us: { + type: 'long', + }, + }, + }, + }, + }, + '@timestamp': { + type: 'date', + }, + }, + }, + }, + }); + + const body = { + params: { + threshold: 30, + windowSize: 5, + windowUnit: 'm', + transactionType: 'request', + environment: 'ENVIRONMENT_ALL', + serviceName: 'opbeans-go', + }, + consumer: 'apm', + alertTypeId: 'apm.transaction_error_rate', + schedule: { interval: '5s' }, + actions: [], + tags: ['apm', 'service.name:opbeans-go'], + notifyWhen: 'onActionGroupChange', + name: 'Transaction error rate threshold | opbeans-go', + }; + + const { body: response, status } = await supertest + .post('/api/alerts/alert') + .send(body) + .set('kbn-xsrf', 'foo'); + + createResponse = { + alert: response, + status, + }; + }); + + after(async () => { + if (createResponse.alert) { + const { body, status } = await supertest + .delete(`/api/alerts/alert/${createResponse.alert.id}`) + .set('kbn-xsrf', 'foo'); + + if (status >= 300) { + const error = new Error('Error deleting alert'); + Object.assign(error, { response: { body, status } }); + throw error; + } + } + + await es.deleteByQuery({ + index: ALERTS_INDEX_TARGET, + body: { + query: { + match_all: {}, + }, + }, + refresh: true, + }); + + await es.indices.delete({ + index: APM_TRANSACTION_INDEX_NAME, + }); + }); + + it('writes alerts data to the alert indices', async () => { + expect(createResponse.status).to.be.below(299); + + expect(createResponse.alert).not.to.be(undefined); + + let alert = await waitUntilNextExecution(createResponse.alert); + + const beforeDataResponse = await es.search({ + index: ALERTS_INDEX_TARGET, + body: { + query: { + match_all: {}, + }, + }, + size: 1, + }); + + expect(beforeDataResponse.body.hits.hits.length).to.be(0); + + await es.index({ + index: APM_TRANSACTION_INDEX_NAME, + body: createTransactionEvent({ + event: { + outcome: 'success', + }, + }), + refresh: true, + }); + + alert = await waitUntilNextExecution(alert); + + const afterInitialDataResponse = await es.search({ + index: ALERTS_INDEX_TARGET, + body: { + query: { + match_all: {}, + }, + }, + size: 1, + }); + + expect(afterInitialDataResponse.body.hits.hits.length).to.be(0); + + await es.index({ + index: APM_TRANSACTION_INDEX_NAME, + body: createTransactionEvent({ + event: { + outcome: 'failure', + }, + }), + refresh: true, + }); + + alert = await waitUntilNextExecution(alert); + + const afterViolatingDataResponse = await es.search({ + index: ALERTS_INDEX_TARGET, + body: { + query: { + match_all: {}, + }, + }, + size: 1, + }); + + expect(afterViolatingDataResponse.body.hits.hits.length).to.be(1); + + const alertEvent = afterViolatingDataResponse.body.hits.hits[0]._source as Record< + string, + any + >; + + const toCompare = omit( + alertEvent, + '@timestamp', + 'kibana.rac.alert.start', + 'kibana.rac.alert.uuid', + 'rule.uuid' + ); + + expectSnapshot(toCompare).toMatchInline(` + Object { + "event.action": "open", + "event.kind": "state", + "kibana.rac.alert.duration.us": 0, + "kibana.rac.alert.id": "apm.transaction_error_rate_opbeans-go_request", + "kibana.rac.alert.status": "open", + "kibana.rac.producer": "apm", + "rule.category": "Transaction error rate threshold", + "rule.id": "apm.transaction_error_rate", + "rule.name": "Transaction error rate threshold | opbeans-go", + "service.name": "opbeans-go", + "tags": Array [ + "apm", + "service.name:opbeans-go", + ], + "transaction.type": "request", + } + `); + }); + }); + }); + + registry.when('Rule registry with write not enabled', { config: 'basic', archives: [] }, () => { + it('does not bootstrap the apm rule indices', async () => { + const errorOrUndefined = await es.indices + .get({ + index: ALERTS_INDEX_TARGET, + expand_wildcards: 'open', + allow_no_indices: false, + }) + .then(() => {}) + .catch((error) => { + return error.toString(); + }); + + expect(errorOrUndefined).not.to.be(undefined); + + expect(errorOrUndefined).to.be(`ResponseError: index_not_found_exception`); + }); + }); +} diff --git a/x-pack/test/apm_api_integration/tests/index.ts b/x-pack/test/apm_api_integration/tests/index.ts index 7c69d5b996cea..53ec61b8d9b61 100644 --- a/x-pack/test/apm_api_integration/tests/index.ts +++ b/x-pack/test/apm_api_integration/tests/index.ts @@ -24,6 +24,10 @@ export default function apmApiIntegrationTests(providerContext: FtrProviderConte loadTestFile(require.resolve('./alerts/chart_preview')); }); + describe('alerts/rule_registry', function () { + loadTestFile(require.resolve('./alerts/rule_registry')); + }); + describe('correlations/latency_slow_transactions', function () { loadTestFile(require.resolve('./correlations/latency_slow_transactions')); }); From b099a0bba285df6705d320e07defc6e193e958f2 Mon Sep 17 00:00:00 2001 From: Liza Katz Date: Fri, 9 Apr 2021 18:29:15 +0300 Subject: [PATCH 4/8] [Partial Results] Move inspector adapter integration into search source (#96241) * Move inspector adapter integration into search source * docs and ts Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- ...ugin-plugins-data-public.isearchoptions.md | 1 + ...-public.isearchoptions.requestresponder.md | 11 ++++ ...ibana-plugin-plugins-data-public.search.md | 1 - ...ublic.searchsource.getsearchrequestbody.md | 4 +- ...ugin-plugins-data-server.isearchoptions.md | 1 + ...-server.isearchoptions.requestresponder.md | 11 ++++ ...ibana-plugin-plugins-data-server.search.md | 2 - .../search_examples/public/search/app.tsx | 4 +- .../data/common/search/aggs/buckets/terms.ts | 50 ++++++-------- .../esaggs/request_handler.test.ts | 2 +- .../expressions/esaggs/request_handler.ts | 65 +++++++------------ .../common/search/expressions/utils/index.ts | 1 - .../data/common/search/search_source/index.ts | 1 + .../search/search_source/inspect/index.ts | 9 +++ .../inspect/inspector_stats.ts} | 4 +- .../search_source/search_source.test.ts | 64 +++++++++--------- .../search/search_source/search_source.ts | 21 ++++-- src/plugins/data/common/search/types.ts | 3 + src/plugins/data/public/index.ts | 3 - src/plugins/data/public/public.api.md | 36 +++++----- src/plugins/data/server/index.ts | 5 -- src/plugins/data/server/server.api.md | 33 +++++----- .../public/application/angular/discover.js | 21 ++---- .../embeddable/search_embeddable.ts | 29 +++------ .../discover/public/kibana_services.ts | 2 +- .../es_geo_grid_source/es_geo_grid_source.tsx | 2 +- .../es_search_source/es_search_source.tsx | 2 +- .../classes/sources/es_source/es_source.ts | 49 ++++---------- .../generate_csv/generate_csv.ts | 2 +- 29 files changed, 205 insertions(+), 234 deletions(-) create mode 100644 docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.requestresponder.md create mode 100644 docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.requestresponder.md create mode 100644 src/plugins/data/common/search/search_source/inspect/index.ts rename src/plugins/data/common/search/{expressions/utils/courier_inspector_stats.ts => search_source/inspect/inspector_stats.ts} (97%) diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.md index 2473c9cfdde8d..cc0cb538be611 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.md @@ -19,6 +19,7 @@ export interface ISearchOptions | [isRestore](./kibana-plugin-plugins-data-public.isearchoptions.isrestore.md) | boolean | Whether the session is restored (i.e. search requests should re-use the stored search IDs, rather than starting from scratch) | | [isStored](./kibana-plugin-plugins-data-public.isearchoptions.isstored.md) | boolean | Whether the session is already saved (i.e. sent to background) | | [legacyHitsTotal](./kibana-plugin-plugins-data-public.isearchoptions.legacyhitstotal.md) | boolean | Request the legacy format for the total number of hits. If sending rest_total_hits_as_int to something other than true, this should be set to false. | +| [requestResponder](./kibana-plugin-plugins-data-public.isearchoptions.requestresponder.md) | RequestResponder | | | [sessionId](./kibana-plugin-plugins-data-public.isearchoptions.sessionid.md) | string | A session ID, grouping multiple search requests into a single session. | | [strategy](./kibana-plugin-plugins-data-public.isearchoptions.strategy.md) | string | Use this option to force using a specific server side search strategy. Leave empty to use the default strategy. | diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.requestresponder.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.requestresponder.md new file mode 100644 index 0000000000000..b4431b9467b71 --- /dev/null +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.requestresponder.md @@ -0,0 +1,11 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) > [ISearchOptions](./kibana-plugin-plugins-data-public.isearchoptions.md) > [requestResponder](./kibana-plugin-plugins-data-public.isearchoptions.requestresponder.md) + +## ISearchOptions.requestResponder property + +Signature: + +```typescript +requestResponder?: RequestResponder; +``` diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.search.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.search.md index cfaad01c029ea..259009c1c5668 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.search.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.search.md @@ -53,7 +53,6 @@ search: { timeRange: import("../common").TimeRange | undefined; } | undefined; }; - getRequestInspectorStats: typeof getRequestInspectorStats; getResponseInspectorStats: typeof getResponseInspectorStats; tabifyAggResponse: typeof tabifyAggResponse; tabifyGetColumns: typeof tabifyGetColumns; diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.getsearchrequestbody.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.getsearchrequestbody.md index cc50d3f017971..d384b9659dbcd 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.getsearchrequestbody.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.getsearchrequestbody.md @@ -9,9 +9,9 @@ Returns body contents of the search request, often referred as query DSL. Signature: ```typescript -getSearchRequestBody(): Promise; +getSearchRequestBody(): any; ``` Returns: -`Promise` +`any` diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.md index 7fd4dd5b8e566..413a59be3d427 100644 --- a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.md +++ b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.md @@ -19,6 +19,7 @@ export interface ISearchOptions | [isRestore](./kibana-plugin-plugins-data-server.isearchoptions.isrestore.md) | boolean | Whether the session is restored (i.e. search requests should re-use the stored search IDs, rather than starting from scratch) | | [isStored](./kibana-plugin-plugins-data-server.isearchoptions.isstored.md) | boolean | Whether the session is already saved (i.e. sent to background) | | [legacyHitsTotal](./kibana-plugin-plugins-data-server.isearchoptions.legacyhitstotal.md) | boolean | Request the legacy format for the total number of hits. If sending rest_total_hits_as_int to something other than true, this should be set to false. | +| [requestResponder](./kibana-plugin-plugins-data-server.isearchoptions.requestresponder.md) | RequestResponder | | | [sessionId](./kibana-plugin-plugins-data-server.isearchoptions.sessionid.md) | string | A session ID, grouping multiple search requests into a single session. | | [strategy](./kibana-plugin-plugins-data-server.isearchoptions.strategy.md) | string | Use this option to force using a specific server side search strategy. Leave empty to use the default strategy. | diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.requestresponder.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.requestresponder.md new file mode 100644 index 0000000000000..7440f5a9d26cf --- /dev/null +++ b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.requestresponder.md @@ -0,0 +1,11 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-data-server](./kibana-plugin-plugins-data-server.md) > [ISearchOptions](./kibana-plugin-plugins-data-server.isearchoptions.md) > [requestResponder](./kibana-plugin-plugins-data-server.isearchoptions.requestresponder.md) + +## ISearchOptions.requestResponder property + +Signature: + +```typescript +requestResponder?: RequestResponder; +``` diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.search.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.search.md index 0911c3e86964d..930f7710f9a00 100644 --- a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.search.md +++ b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.search.md @@ -36,8 +36,6 @@ search: { toAbsoluteDates: typeof toAbsoluteDates; calcAutoIntervalLessThan: typeof calcAutoIntervalLessThan; }; - getRequestInspectorStats: typeof getRequestInspectorStats; - getResponseInspectorStats: typeof getResponseInspectorStats; tabifyAggResponse: typeof tabifyAggResponse; tabifyGetColumns: typeof tabifyGetColumns; } diff --git a/examples/search_examples/public/search/app.tsx b/examples/search_examples/public/search/app.tsx index c87bf21e0e71c..3bac445581ae7 100644 --- a/examples/search_examples/public/search/app.tsx +++ b/examples/search_examples/public/search/app.tsx @@ -204,8 +204,8 @@ export const SearchExamplesApp = ({ }); } - setRequest(await searchSource.getSearchRequestBody()); - const res = await searchSource.fetch(); + setRequest(searchSource.getSearchRequestBody()); + const res = await searchSource.fetch$().toPromise(); setResponse(res); const message = Searched {res.hits.total} documents.; diff --git a/src/plugins/data/common/search/aggs/buckets/terms.ts b/src/plugins/data/common/search/aggs/buckets/terms.ts index 7d37dc83405b8..77c9c6e391c0a 100644 --- a/src/plugins/data/common/search/aggs/buckets/terms.ts +++ b/src/plugins/data/common/search/aggs/buckets/terms.ts @@ -8,7 +8,6 @@ import { noop } from 'lodash'; import { i18n } from '@kbn/i18n'; -import type { RequestAdapter } from 'src/plugins/inspector/common'; import { BucketAggType, IBucketAggConfig } from './bucket_agg_type'; import { BUCKET_TYPES } from './bucket_agg_types'; @@ -21,7 +20,6 @@ import { aggTermsFnName } from './terms_fn'; import { AggConfigSerialized, BaseAggParams } from '../types'; import { KBN_FIELD_TYPES } from '../../../../common'; -import { getRequestInspectorStats, getResponseInspectorStats } from '../../expressions'; import { buildOtherBucketAgg, @@ -103,36 +101,28 @@ export const getTermsBucketAgg = () => nestedSearchSource.setField('aggs', filterAgg); - let request: ReturnType | undefined; - if (inspectorRequestAdapter) { - request = inspectorRequestAdapter.start( - i18n.translate('data.search.aggs.buckets.terms.otherBucketTitle', { - defaultMessage: 'Other bucket', + const requestResponder = inspectorRequestAdapter?.start( + i18n.translate('data.search.aggs.buckets.terms.otherBucketTitle', { + defaultMessage: 'Other bucket', + }), + { + description: i18n.translate('data.search.aggs.buckets.terms.otherBucketDescription', { + defaultMessage: + 'This request counts the number of documents that fall ' + + 'outside the criterion of the data buckets.', }), - { - description: i18n.translate('data.search.aggs.buckets.terms.otherBucketDescription', { - defaultMessage: - 'This request counts the number of documents that fall ' + - 'outside the criterion of the data buckets.', - }), - searchSessionId, - } - ); - nestedSearchSource.getSearchRequestBody().then((body) => { - request!.json(body); - }); - request.stats(getRequestInspectorStats(nestedSearchSource)); - } + searchSessionId, + } + ); + + const response = await nestedSearchSource + .fetch$({ + abortSignal, + sessionId: searchSessionId, + requestResponder, + }) + .toPromise(); - const response = await nestedSearchSource.fetch({ - abortSignal, - sessionId: searchSessionId, - }); - if (request) { - request - .stats(getResponseInspectorStats(response, nestedSearchSource)) - .ok({ json: response }); - } resp = mergeOtherBucketAggResponse(aggConfigs, resp, response, aggConfig, filterAgg()); } if (aggConfig.params.missingBucket) { diff --git a/src/plugins/data/common/search/expressions/esaggs/request_handler.test.ts b/src/plugins/data/common/search/expressions/esaggs/request_handler.test.ts index 7580032b0dd85..c2566535916a8 100644 --- a/src/plugins/data/common/search/expressions/esaggs/request_handler.test.ts +++ b/src/plugins/data/common/search/expressions/esaggs/request_handler.test.ts @@ -133,7 +133,7 @@ describe('esaggs expression function - public', () => { test('calls searchSource.fetch', async () => { await handleRequest(mockParams); const searchSource = await mockParams.searchSourceService.create(); - expect(searchSource.fetch).toHaveBeenCalledWith({ + expect(searchSource.fetch$).toHaveBeenCalledWith({ abortSignal: mockParams.abortSignal, sessionId: mockParams.searchSessionId, }); diff --git a/src/plugins/data/common/search/expressions/esaggs/request_handler.ts b/src/plugins/data/common/search/expressions/esaggs/request_handler.ts index 72d9cc4095570..5620698a47538 100644 --- a/src/plugins/data/common/search/expressions/esaggs/request_handler.ts +++ b/src/plugins/data/common/search/expressions/esaggs/request_handler.ts @@ -22,7 +22,6 @@ import { import { IAggConfigs } from '../../aggs'; import { ISearchStartSearchSource } from '../../search_source'; import { tabifyAggResponse } from '../../tabify'; -import { getRequestInspectorStats, getResponseInspectorStats } from '../utils'; /** @internal */ export interface RequestHandlerParams { @@ -41,6 +40,21 @@ export interface RequestHandlerParams { getNow?: () => Date; } +function getRequestMainResponder(inspectorAdapters: Adapters, searchSessionId?: string) { + return inspectorAdapters.requests?.start( + i18n.translate('data.functions.esaggs.inspector.dataRequest.title', { + defaultMessage: 'Data', + }), + { + description: i18n.translate('data.functions.esaggs.inspector.dataRequest.description', { + defaultMessage: + 'This request queries Elasticsearch to fetch the data for the visualization.', + }), + searchSessionId, + } + ); +} + export const handleRequest = async ({ abortSignal, aggs, @@ -113,52 +127,19 @@ export const handleRequest = async ({ requestSearchSource.setField('filter', filters); requestSearchSource.setField('query', query); - let request; - if (inspectorAdapters.requests) { - inspectorAdapters.requests.reset(); - request = inspectorAdapters.requests.start( - i18n.translate('data.functions.esaggs.inspector.dataRequest.title', { - defaultMessage: 'Data', - }), - { - description: i18n.translate('data.functions.esaggs.inspector.dataRequest.description', { - defaultMessage: - 'This request queries Elasticsearch to fetch the data for the visualization.', - }), - searchSessionId, - } - ); - request.stats(getRequestInspectorStats(requestSearchSource)); - } - - try { - const response = await requestSearchSource.fetch({ - abortSignal, - sessionId: searchSessionId, - }); - - if (request) { - request.stats(getResponseInspectorStats(response, searchSource)).ok({ json: response }); - } + inspectorAdapters.requests?.reset(); + const requestResponder = getRequestMainResponder(inspectorAdapters, searchSessionId); - (searchSource as any).rawResponse = response; - } catch (e) { - // Log any error during request to the inspector - if (request) { - request.error({ json: e }); - } - throw e; - } finally { - // Add the request body no matter if things went fine or not - if (request) { - request.json(await requestSearchSource.getSearchRequestBody()); - } - } + const response$ = await requestSearchSource.fetch$({ + abortSignal, + sessionId: searchSessionId, + requestResponder, + }); // Note that rawResponse is not deeply cloned here, so downstream applications using courier // must take care not to mutate it, or it could have unintended side effects, e.g. displaying // response data incorrectly in the inspector. - let response = (searchSource as any).rawResponse; + let response = await response$.toPromise(); for (const agg of aggs.aggs) { if (agg.enabled && typeof agg.type.postFlightRequest === 'function') { response = await agg.type.postFlightRequest( diff --git a/src/plugins/data/common/search/expressions/utils/index.ts b/src/plugins/data/common/search/expressions/utils/index.ts index 2fa54d47445b3..a6ea8da6ac6e9 100644 --- a/src/plugins/data/common/search/expressions/utils/index.ts +++ b/src/plugins/data/common/search/expressions/utils/index.ts @@ -6,5 +6,4 @@ * Side Public License, v 1. */ -export * from './courier_inspector_stats'; export * from './function_wrapper'; diff --git a/src/plugins/data/common/search/search_source/index.ts b/src/plugins/data/common/search/search_source/index.ts index 1cb04075dad7a..757e0de6ecb49 100644 --- a/src/plugins/data/common/search/search_source/index.ts +++ b/src/plugins/data/common/search/search_source/index.ts @@ -10,6 +10,7 @@ export { createSearchSource } from './create_search_source'; export { injectReferences } from './inject_references'; export { extractReferences } from './extract_references'; export { parseSearchSourceJSON } from './parse_json'; +export { getResponseInspectorStats } from './inspect'; export * from './fetch'; export * from './legacy'; export * from './search_source'; diff --git a/src/plugins/data/common/search/search_source/inspect/index.ts b/src/plugins/data/common/search/search_source/inspect/index.ts new file mode 100644 index 0000000000000..d5947f8a18cc9 --- /dev/null +++ b/src/plugins/data/common/search/search_source/inspect/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export * from './inspector_stats'; diff --git a/src/plugins/data/common/search/expressions/utils/courier_inspector_stats.ts b/src/plugins/data/common/search/search_source/inspect/inspector_stats.ts similarity index 97% rename from src/plugins/data/common/search/expressions/utils/courier_inspector_stats.ts rename to src/plugins/data/common/search/search_source/inspect/inspector_stats.ts index 99acbce8935c4..24507a7e13058 100644 --- a/src/plugins/data/common/search/expressions/utils/courier_inspector_stats.ts +++ b/src/plugins/data/common/search/search_source/inspect/inspector_stats.ts @@ -15,8 +15,8 @@ import { i18n } from '@kbn/i18n'; import type { estypes } from '@elastic/elasticsearch'; -import { ISearchSource } from 'src/plugins/data/public'; -import { RequestStatistics } from 'src/plugins/inspector/common'; +import type { ISearchSource } from 'src/plugins/data/public'; +import type { RequestStatistics } from 'src/plugins/inspector/common'; /** @public */ export function getRequestInspectorStats(searchSource: ISearchSource) { diff --git a/src/plugins/data/common/search/search_source/search_source.test.ts b/src/plugins/data/common/search/search_source/search_source.test.ts index fd97a3d3381a9..3726e5d0c33e8 100644 --- a/src/plugins/data/common/search/search_source/search_source.test.ts +++ b/src/plugins/data/common/search/search_source/search_source.test.ts @@ -125,7 +125,7 @@ describe('SearchSource', () => { }), } as unknown) as IndexPattern); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request.stored_fields).toEqual(['hello']); expect(request.script_fields).toEqual({ world: {} }); expect(request.fields).toEqual(['@timestamp']); @@ -144,7 +144,7 @@ describe('SearchSource', () => { searchSource.setField('fields', ['@timestamp']); searchSource.setField('fieldsFromSource', ['foo']); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request).not.toHaveProperty('docvalue_fields'); }); @@ -160,7 +160,7 @@ describe('SearchSource', () => { // @ts-expect-error TS won't like using this field name, but technically it's possible. searchSource.setField('docvalue_fields', ['world']); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request).toHaveProperty('docvalue_fields'); expect(request.docvalue_fields).toEqual(['world']); }); @@ -179,7 +179,7 @@ describe('SearchSource', () => { searchSource.setField('fields', ['c']); searchSource.setField('fieldsFromSource', ['a', 'b', 'd']); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request).toHaveProperty('docvalue_fields'); expect(request._source.includes).toEqual(['c', 'a', 'b', 'd']); expect(request.docvalue_fields).toEqual([{ field: 'b', format: 'date_time' }]); @@ -202,7 +202,7 @@ describe('SearchSource', () => { } as unknown) as IndexPattern); searchSource.setField('fields', [{ field: 'hello', format: 'strict_date_time' }]); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request).toHaveProperty('fields'); expect(request.fields).toEqual([{ field: 'hello', format: 'strict_date_time' }]); }); @@ -218,7 +218,7 @@ describe('SearchSource', () => { } as unknown) as IndexPattern); searchSource.setField('fields', ['hello']); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request).toHaveProperty('fields'); expect(request.fields).toEqual([{ field: 'hello', format: 'date_time' }]); }); @@ -239,7 +239,7 @@ describe('SearchSource', () => { } as unknown) as IndexPattern); searchSource.setField('fields', [{ field: 'hello', a: 'a', c: 'c' }]); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request).toHaveProperty('fields'); expect(request.fields).toEqual([ { field: 'hello', format: 'date_time', a: 'a', b: 'test', c: 'c' }, @@ -258,7 +258,7 @@ describe('SearchSource', () => { // @ts-expect-error TS won't like using this field name, but technically it's possible. searchSource.setField('script_fields', { world: {} }); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request).toHaveProperty('script_fields'); expect(request.script_fields).toEqual({ hello: {}, @@ -277,7 +277,7 @@ describe('SearchSource', () => { } as unknown) as IndexPattern); searchSource.setField('fields', ['hello', 'a', { field: 'c' }]); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request.script_fields).toEqual({ hello: {} }); expect(request.stored_fields).toEqual(['a', 'c']); }); @@ -293,7 +293,7 @@ describe('SearchSource', () => { } as unknown) as IndexPattern); searchSource.setField('fields', ['hello', 'a', { foo: 'c' }]); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request.script_fields).toEqual({ hello: {} }); expect(request.stored_fields).toEqual(['a']); }); @@ -309,23 +309,23 @@ describe('SearchSource', () => { } as unknown) as IndexPattern); searchSource.setField('fieldsFromSource', ['hello', 'a']); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request.script_fields).toEqual({ hello: {} }); expect(request.stored_fields).toEqual(['a']); }); test('defaults to * for stored fields when no fields are provided', async () => { - const requestA = await searchSource.getSearchRequestBody(); + const requestA = searchSource.getSearchRequestBody(); expect(requestA.stored_fields).toEqual(['*']); searchSource.setField('fields', ['*']); - const requestB = await searchSource.getSearchRequestBody(); + const requestB = searchSource.getSearchRequestBody(); expect(requestB.stored_fields).toEqual(['*']); }); test('defaults to * for stored fields when no fields are provided with fieldsFromSource', async () => { searchSource.setField('fieldsFromSource', ['*']); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request.stored_fields).toEqual(['*']); }); }); @@ -343,7 +343,7 @@ describe('SearchSource', () => { // @ts-expect-error Typings for excludes filters need to be fixed. searchSource.setField('source', { excludes: ['exclude-*'] }); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request.fields).toEqual(['@timestamp']); }); @@ -357,7 +357,7 @@ describe('SearchSource', () => { }), } as unknown) as IndexPattern); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request.fields).toEqual(['@timestamp']); }); @@ -372,7 +372,7 @@ describe('SearchSource', () => { } as unknown) as IndexPattern); searchSource.setField('fields', ['hello']); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request.script_fields).toEqual({ hello: {} }); }); @@ -387,7 +387,7 @@ describe('SearchSource', () => { } as unknown) as IndexPattern); searchSource.setField('fields', ['hello', 'foo']); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request.fields).toEqual(['hello']); }); @@ -402,7 +402,7 @@ describe('SearchSource', () => { } as unknown) as IndexPattern); searchSource.setField('fields', ['*']); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request.fields).toEqual([{ field: 'field1' }, { field: 'field2' }]); }); @@ -417,7 +417,7 @@ describe('SearchSource', () => { } as unknown) as IndexPattern); searchSource.setField('fields', [{ field: '*', include_unmapped: 'true' }]); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request.fields).toEqual([{ field: 'field1' }, { field: 'field2' }]); }); @@ -432,7 +432,7 @@ describe('SearchSource', () => { } as unknown) as IndexPattern); searchSource.setField('fields', ['timestamp', '*']); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request.script_fields).toEqual({ hello: {}, world: {} }); }); }); @@ -455,7 +455,7 @@ describe('SearchSource', () => { 'bar-b', ]); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request._source).toEqual({ includes: ['@timestamp', 'bar-b'], }); @@ -473,7 +473,7 @@ describe('SearchSource', () => { } as unknown) as IndexPattern); searchSource.setField('fields', ['hello', '@timestamp', 'foo-a', 'bar']); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request.fields).toEqual(['hello', '@timestamp', 'bar', 'date']); expect(request.script_fields).toEqual({ hello: {} }); expect(request.stored_fields).toEqual(['@timestamp', 'bar']); @@ -498,7 +498,7 @@ describe('SearchSource', () => { 'runtime_field', ]); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request._source).toEqual({ includes: ['@timestamp', 'bar'], }); @@ -520,7 +520,7 @@ describe('SearchSource', () => { searchSource.setField('fields', ['hello', '@timestamp', 'foo-a', 'bar']); searchSource.setField('fieldsFromSource', ['foo-b', 'date', 'baz']); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request._source).toEqual({ includes: ['@timestamp', 'bar', 'date', 'baz'], }); @@ -546,7 +546,7 @@ describe('SearchSource', () => { } as unknown) as IndexPattern); searchSource.setField('fields', ['*']); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request.fields).toEqual([ '*', { field: '@timestamp', format: 'strict_date_optional_time_nanos' }, @@ -574,7 +574,7 @@ describe('SearchSource', () => { } as unknown) as IndexPattern); searchSource.setField('fields', ['*']); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request.fields).toEqual([ { field: 'foo-bar' }, { field: 'field1' }, @@ -592,14 +592,14 @@ describe('SearchSource', () => { expect(searchSource.getField('source')).toBe(undefined); searchSource.setField('index', indexPattern); expect(searchSource.getField('index')).toBe(indexPattern); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request._source).toBe(mockSource); }); test('removes created searchSource filter on removal', async () => { searchSource.setField('index', indexPattern); searchSource.setField('index', undefined); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request._source).toBe(undefined); }); }); @@ -609,7 +609,7 @@ describe('SearchSource', () => { searchSource.setField('index', indexPattern); searchSource.setField('index', indexPattern2); expect(searchSource.getField('index')).toBe(indexPattern2); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request._source).toBe(mockSource2); }); @@ -617,7 +617,7 @@ describe('SearchSource', () => { searchSource.setField('index', indexPattern); searchSource.setField('index', indexPattern2); searchSource.setField('index', undefined); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request._source).toBe(undefined); }); }); @@ -808,7 +808,7 @@ describe('SearchSource', () => { docvalueFields: [], }), } as unknown) as IndexPattern); - const request = await searchSource.getSearchRequestBody(); + const request = searchSource.getSearchRequestBody(); expect(request.stored_fields).toEqual(['geometry', 'prop1']); expect(request.docvalue_fields).toEqual(['prop1']); expect(request._source).toEqual(['geometry']); diff --git a/src/plugins/data/common/search/search_source/search_source.ts b/src/plugins/data/common/search/search_source/search_source.ts index f11e7f06b6ab9..e1e7a8292d677 100644 --- a/src/plugins/data/common/search/search_source/search_source.ts +++ b/src/plugins/data/common/search/search_source/search_source.ts @@ -60,7 +60,7 @@ import { setWith } from '@elastic/safer-lodash-set'; import { uniqueId, keyBy, pick, difference, isFunction, isEqual, uniqWith, isObject } from 'lodash'; -import { map, switchMap, tap } from 'rxjs/operators'; +import { catchError, finalize, map, switchMap, tap } from 'rxjs/operators'; import { defer, from } from 'rxjs'; import { normalizeSortRequest } from './normalize_sort_request'; import { fieldWildcardFilter } from '../../../../kibana_utils/common'; @@ -73,6 +73,7 @@ import type { SearchSourceFields, } from './types'; import { FetchHandlers, RequestFailure, getSearchParamsFromRequest, SearchRequest } from './fetch'; +import { getRequestInspectorStats, getResponseInspectorStats } from './inspect'; import { getEsQueryConfig, buildEsQuery, Filter, UI_SETTINGS } from '../../../common'; import { getHighlightRequest } from '../../../common/field_formats'; @@ -256,6 +257,9 @@ export class SearchSource { fetch$(options: ISearchOptions = {}) { const { getConfig } = this.dependencies; return defer(() => this.requestIsStarting(options)).pipe( + tap(() => { + options.requestResponder?.stats(getRequestInspectorStats(this)); + }), switchMap(() => { const searchRequest = this.flatten(); this.history = [searchRequest]; @@ -271,7 +275,17 @@ export class SearchSource { // TODO: Remove casting when https://github.com/elastic/elasticsearch-js/issues/1287 is resolved if ((response as any).error) { throw new RequestFailure(null, response); + } else { + options.requestResponder?.stats(getResponseInspectorStats(response, this)); + options.requestResponder?.ok({ json: response }); } + }), + catchError((e) => { + options.requestResponder?.error({ json: e }); + throw e; + }), + finalize(() => { + options.requestResponder?.json(this.getSearchRequestBody()); }) ); } @@ -298,9 +312,8 @@ export class SearchSource { /** * Returns body contents of the search request, often referred as query DSL. */ - async getSearchRequestBody() { - const searchRequest = await this.flatten(); - return searchRequest.body; + getSearchRequestBody() { + return this.flatten().body; } /** diff --git a/src/plugins/data/common/search/types.ts b/src/plugins/data/common/search/types.ts index d77a2ea62bb9a..37de8dc49d3c6 100644 --- a/src/plugins/data/common/search/types.ts +++ b/src/plugins/data/common/search/types.ts @@ -9,6 +9,7 @@ import { Observable } from 'rxjs'; import { IEsSearchRequest, IEsSearchResponse } from './es_search'; import { IndexPattern } from '..'; +import type { RequestResponder } from '../../../inspector/common'; export type ISearchGeneric = < SearchStrategyRequest extends IKibanaSearchRequest = IEsSearchRequest, @@ -118,6 +119,8 @@ export interface ISearchOptions { */ indexPattern?: IndexPattern; + + requestResponder?: RequestResponder; } /** diff --git a/src/plugins/data/public/index.ts b/src/plugins/data/public/index.ts index d2683e248b7bf..e86b64d135d59 100644 --- a/src/plugins/data/public/index.ts +++ b/src/plugins/data/public/index.ts @@ -314,8 +314,6 @@ import { boundsDescendingRaw, getNumberHistogramIntervalByDatatableColumn, getDateHistogramMetaDataByDatatableColumn, - // expressions utils - getRequestInspectorStats, getResponseInspectorStats, // tabify tabifyAggResponse, @@ -428,7 +426,6 @@ export const search = { getNumberHistogramIntervalByDatatableColumn, getDateHistogramMetaDataByDatatableColumn, }, - getRequestInspectorStats, getResponseInspectorStats, tabifyAggResponse, tabifyGetColumns, diff --git a/src/plugins/data/public/public.api.md b/src/plugins/data/public/public.api.md index c80f008636ba6..c4e54c64af132 100644 --- a/src/plugins/data/public/public.api.md +++ b/src/plugins/data/public/public.api.md @@ -1676,6 +1676,10 @@ export interface ISearchOptions { isRestore?: boolean; isStored?: boolean; legacyHitsTotal?: boolean; + // Warning: (ae-forgotten-export) The symbol "RequestResponder" needs to be exported by the entry point index.d.ts + // + // (undocumented) + requestResponder?: RequestResponder; sessionId?: string; strategy?: string; } @@ -2298,7 +2302,6 @@ export const search: { timeRange: import("../common").TimeRange | undefined; } | undefined; }; - getRequestInspectorStats: typeof getRequestInspectorStats; getResponseInspectorStats: typeof getResponseInspectorStats; tabifyAggResponse: typeof tabifyAggResponse; tabifyGetColumns: typeof tabifyGetColumns; @@ -2434,7 +2437,7 @@ export class SearchSource { getId(): string; getOwnField(field: K): SearchSourceFields[K]; getParent(): SearchSource | undefined; - getSearchRequestBody(): Promise; + getSearchRequestBody(): any; getSerializedFields(recurse?: boolean): SearchSourceFields; // Warning: (ae-incompatible-release-tags) The symbol "history" is marked as @public, but its signature references "SearchRequest" which is marked as @internal // @@ -2712,21 +2715,20 @@ export interface WaitUntilNextSessionCompletesOptions { // src/plugins/data/public/index.ts:238:27 - (ae-forgotten-export) The symbol "validateIndexPattern" needs to be exported by the entry point index.d.ts // src/plugins/data/public/index.ts:238:27 - (ae-forgotten-export) The symbol "flattenHitWrapper" needs to be exported by the entry point index.d.ts // src/plugins/data/public/index.ts:238:27 - (ae-forgotten-export) The symbol "formatHitProvider" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:406:20 - (ae-forgotten-export) The symbol "getRequestInspectorStats" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:406:20 - (ae-forgotten-export) The symbol "getResponseInspectorStats" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:406:20 - (ae-forgotten-export) The symbol "tabifyAggResponse" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:406:20 - (ae-forgotten-export) The symbol "tabifyGetColumns" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:408:1 - (ae-forgotten-export) The symbol "CidrMask" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:409:1 - (ae-forgotten-export) The symbol "dateHistogramInterval" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:418:1 - (ae-forgotten-export) The symbol "InvalidEsCalendarIntervalError" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:419:1 - (ae-forgotten-export) The symbol "InvalidEsIntervalFormatError" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:420:1 - (ae-forgotten-export) The symbol "Ipv4Address" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:421:1 - (ae-forgotten-export) The symbol "isDateHistogramBucketAggConfig" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:425:1 - (ae-forgotten-export) The symbol "isValidEsInterval" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:426:1 - (ae-forgotten-export) The symbol "isValidInterval" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:429:1 - (ae-forgotten-export) The symbol "parseInterval" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:430:1 - (ae-forgotten-export) The symbol "propFilter" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:433:1 - (ae-forgotten-export) The symbol "toAbsoluteDates" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:404:20 - (ae-forgotten-export) The symbol "getResponseInspectorStats" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:404:20 - (ae-forgotten-export) The symbol "tabifyAggResponse" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:404:20 - (ae-forgotten-export) The symbol "tabifyGetColumns" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:406:1 - (ae-forgotten-export) The symbol "CidrMask" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:407:1 - (ae-forgotten-export) The symbol "dateHistogramInterval" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:416:1 - (ae-forgotten-export) The symbol "InvalidEsCalendarIntervalError" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:417:1 - (ae-forgotten-export) The symbol "InvalidEsIntervalFormatError" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:418:1 - (ae-forgotten-export) The symbol "Ipv4Address" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:419:1 - (ae-forgotten-export) The symbol "isDateHistogramBucketAggConfig" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:423:1 - (ae-forgotten-export) The symbol "isValidEsInterval" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:424:1 - (ae-forgotten-export) The symbol "isValidInterval" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:427:1 - (ae-forgotten-export) The symbol "parseInterval" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:428:1 - (ae-forgotten-export) The symbol "propFilter" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:431:1 - (ae-forgotten-export) The symbol "toAbsoluteDates" needs to be exported by the entry point index.d.ts // src/plugins/data/public/query/state_sync/connect_to_query_state.ts:34:5 - (ae-forgotten-export) The symbol "FilterStateStore" needs to be exported by the entry point index.d.ts // src/plugins/data/public/search/session/session_service.ts:56:5 - (ae-forgotten-export) The symbol "UrlGeneratorStateMapping" needs to be exported by the entry point index.d.ts diff --git a/src/plugins/data/server/index.ts b/src/plugins/data/server/index.ts index cbf09ef57d96a..fa54f45d2feb2 100644 --- a/src/plugins/data/server/index.ts +++ b/src/plugins/data/server/index.ts @@ -176,9 +176,6 @@ import { parseEsInterval, parseInterval, toAbsoluteDates, - // expressions utils - getRequestInspectorStats, - getResponseInspectorStats, // tabify tabifyAggResponse, tabifyGetColumns, @@ -263,8 +260,6 @@ export const search = { toAbsoluteDates, calcAutoIntervalLessThan, }, - getRequestInspectorStats, - getResponseInspectorStats, tabifyAggResponse, tabifyGetColumns, }; diff --git a/src/plugins/data/server/server.api.md b/src/plugins/data/server/server.api.md index 053b60956fa92..0ea3af60e9b5d 100644 --- a/src/plugins/data/server/server.api.md +++ b/src/plugins/data/server/server.api.md @@ -56,7 +56,6 @@ import { PublicMethodsOf } from '@kbn/utility-types'; import { RecursiveReadonly } from '@kbn/utility-types'; import { RequestAdapter } from 'src/plugins/inspector/common'; import { RequestHandlerContext } from 'src/core/server'; -import { RequestStatistics } from 'src/plugins/inspector/common'; import { SavedObject } from 'kibana/server'; import { SavedObject as SavedObject_2 } from 'src/core/server'; import { SavedObjectsClientContract } from 'src/core/server'; @@ -1002,6 +1001,10 @@ export interface ISearchOptions { isRestore?: boolean; isStored?: boolean; legacyHitsTotal?: boolean; + // Warning: (ae-forgotten-export) The symbol "RequestResponder" needs to be exported by the entry point index.d.ts + // + // (undocumented) + requestResponder?: RequestResponder; sessionId?: string; strategy?: string; } @@ -1327,8 +1330,6 @@ export const search: { toAbsoluteDates: typeof toAbsoluteDates; calcAutoIntervalLessThan: typeof calcAutoIntervalLessThan; }; - getRequestInspectorStats: typeof getRequestInspectorStats; - getResponseInspectorStats: typeof getResponseInspectorStats; tabifyAggResponse: typeof tabifyAggResponse; tabifyGetColumns: typeof tabifyGetColumns; }; @@ -1510,20 +1511,18 @@ export function usageProvider(core: CoreSetup_2): SearchUsage; // src/plugins/data/server/index.ts:101:26 - (ae-forgotten-export) The symbol "HistogramFormat" needs to be exported by the entry point index.d.ts // src/plugins/data/server/index.ts:128:27 - (ae-forgotten-export) The symbol "isFilterable" needs to be exported by the entry point index.d.ts // src/plugins/data/server/index.ts:128:27 - (ae-forgotten-export) The symbol "isNestedField" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:244:20 - (ae-forgotten-export) The symbol "getRequestInspectorStats" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:244:20 - (ae-forgotten-export) The symbol "getResponseInspectorStats" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:244:20 - (ae-forgotten-export) The symbol "tabifyAggResponse" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:244:20 - (ae-forgotten-export) The symbol "tabifyGetColumns" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:246:1 - (ae-forgotten-export) The symbol "CidrMask" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:247:1 - (ae-forgotten-export) The symbol "dateHistogramInterval" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:256:1 - (ae-forgotten-export) The symbol "InvalidEsCalendarIntervalError" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:257:1 - (ae-forgotten-export) The symbol "InvalidEsIntervalFormatError" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:258:1 - (ae-forgotten-export) The symbol "Ipv4Address" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:262:1 - (ae-forgotten-export) The symbol "isValidEsInterval" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:263:1 - (ae-forgotten-export) The symbol "isValidInterval" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:267:1 - (ae-forgotten-export) The symbol "propFilter" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:270:1 - (ae-forgotten-export) The symbol "toAbsoluteDates" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:271:1 - (ae-forgotten-export) The symbol "calcAutoIntervalLessThan" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:241:20 - (ae-forgotten-export) The symbol "tabifyAggResponse" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:241:20 - (ae-forgotten-export) The symbol "tabifyGetColumns" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:243:1 - (ae-forgotten-export) The symbol "CidrMask" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:244:1 - (ae-forgotten-export) The symbol "dateHistogramInterval" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:253:1 - (ae-forgotten-export) The symbol "InvalidEsCalendarIntervalError" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:254:1 - (ae-forgotten-export) The symbol "InvalidEsIntervalFormatError" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:255:1 - (ae-forgotten-export) The symbol "Ipv4Address" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:259:1 - (ae-forgotten-export) The symbol "isValidEsInterval" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:260:1 - (ae-forgotten-export) The symbol "isValidInterval" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:264:1 - (ae-forgotten-export) The symbol "propFilter" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:267:1 - (ae-forgotten-export) The symbol "toAbsoluteDates" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:268:1 - (ae-forgotten-export) The symbol "calcAutoIntervalLessThan" needs to be exported by the entry point index.d.ts // src/plugins/data/server/plugin.ts:81:74 - (ae-forgotten-export) The symbol "DataEnhancements" needs to be exported by the entry point index.d.ts // src/plugins/data/server/search/types.ts:114:5 - (ae-forgotten-export) The symbol "ISearchStartSearchSource" needs to be exported by the entry point index.d.ts diff --git a/src/plugins/discover/public/application/angular/discover.js b/src/plugins/discover/public/application/angular/discover.js index 3be047859d3b0..45382af098644 100644 --- a/src/plugins/discover/public/application/angular/discover.js +++ b/src/plugins/discover/public/application/angular/discover.js @@ -25,8 +25,6 @@ import { discoverResponseHandler } from './response_handler'; import { getAngularModule, getHeaderActionMenuMounter, - getRequestInspectorStats, - getResponseInspectorStats, getServices, getUrlTracker, redirectWhenMissing, @@ -153,7 +151,6 @@ function discoverController($route, $scope) { const subscriptions = new Subscription(); const refetch$ = new Subject(); - let inspectorRequest; let isChangingIndexPattern = false; const savedSearch = $route.current.locals.savedObjects.savedSearch; const persistentSearchSource = savedSearch.searchSource; @@ -417,12 +414,14 @@ function discoverController($route, $scope) { $scope.fetchStatus = fetchStatuses.LOADING; $scope.resultState = getResultState($scope.fetchStatus, $scope.rows); - logInspectorRequest({ searchSessionId }); + return $scope.volatileSearchSource - .fetch({ + .fetch$({ abortSignal: abortController.signal, sessionId: searchSessionId, + requestResponder: getRequestResponder({ searchSessionId }), }) + .toPromise() .then(onResults) .catch((error) => { // If the request was aborted then no need to surface this error in the UI @@ -439,10 +438,6 @@ function discoverController($route, $scope) { }; function onResults(resp) { - inspectorRequest - .stats(getResponseInspectorStats(resp, $scope.volatileSearchSource)) - .ok({ json: resp }); - if (getTimeField() && !$scope.state.hideChart) { const tabifiedData = tabifyAggResponse($scope.opts.chartAggConfigs, resp); $scope.volatileSearchSource.rawResponse = resp; @@ -463,7 +458,7 @@ function discoverController($route, $scope) { $scope.fetchStatus = fetchStatuses.COMPLETE; } - function logInspectorRequest({ searchSessionId = null } = { searchSessionId: null }) { + function getRequestResponder({ searchSessionId = null } = { searchSessionId: null }) { inspectorAdapters.requests.reset(); const title = i18n.translate('discover.inspectorRequestDataTitle', { defaultMessage: 'data', @@ -471,11 +466,7 @@ function discoverController($route, $scope) { const description = i18n.translate('discover.inspectorRequestDescription', { defaultMessage: 'This request queries Elasticsearch to fetch the data for the search.', }); - inspectorRequest = inspectorAdapters.requests.start(title, { description, searchSessionId }); - inspectorRequest.stats(getRequestInspectorStats($scope.volatileSearchSource)); - $scope.volatileSearchSource.getSearchRequestBody().then((body) => { - inspectorRequest.json(body); - }); + return inspectorAdapters.requests.start(title, { description, searchSessionId }); } $scope.resetQuery = function () { diff --git a/src/plugins/discover/public/application/embeddable/search_embeddable.ts b/src/plugins/discover/public/application/embeddable/search_embeddable.ts index e7349ed22355a..237da72ae3a52 100644 --- a/src/plugins/discover/public/application/embeddable/search_embeddable.ts +++ b/src/plugins/discover/public/application/embeddable/search_embeddable.ts @@ -29,13 +29,7 @@ import searchTemplateGrid from './search_template_datagrid.html'; import { ISearchEmbeddable, SearchInput, SearchOutput } from './types'; import { SortOrder } from '../angular/doc_table/components/table_header/helpers'; import { getSortForSearchSource } from '../angular/doc_table'; -import { - getRequestInspectorStats, - getResponseInspectorStats, - getServices, - IndexPattern, - ISearchSource, -} from '../../kibana_services'; +import { getServices, IndexPattern, ISearchSource } from '../../kibana_services'; import { SEARCH_EMBEDDABLE_TYPE } from './constants'; import { SavedSearch } from '../..'; import { @@ -330,14 +324,11 @@ export class SearchEmbeddable defaultMessage: 'This request queries Elasticsearch to fetch the data for the search.', }); - const inspectorRequest = this.inspectorAdapters.requests!.start(title, { + const requestResponder = this.inspectorAdapters.requests!.start(title, { description, searchSessionId, }); - inspectorRequest.stats(getRequestInspectorStats(searchSource)); - searchSource.getSearchRequestBody().then((body: Record) => { - inspectorRequest.json(body); - }); + this.searchScope.$apply(() => { this.searchScope!.isLoading = true; }); @@ -345,15 +336,15 @@ export class SearchEmbeddable try { // Make the request - const resp = await searchSource.fetch({ - abortSignal: this.abortController.signal, - sessionId: searchSessionId, - }); + const resp = await searchSource + .fetch$({ + abortSignal: this.abortController.signal, + sessionId: searchSessionId, + requestResponder, + }) + .toPromise(); this.updateOutput({ loading: false, error: undefined }); - // Log response to inspector - inspectorRequest.stats(getResponseInspectorStats(resp, searchSource)).ok({ json: resp }); - // Apply the changes to the angular scope this.searchScope.$apply(() => { this.searchScope!.hits = resp.hits.hits; diff --git a/src/plugins/discover/public/kibana_services.ts b/src/plugins/discover/public/kibana_services.ts index 27bcc00234939..e4b0035ed0e03 100644 --- a/src/plugins/discover/public/kibana_services.ts +++ b/src/plugins/discover/public/kibana_services.ts @@ -88,7 +88,7 @@ export const [getScopedHistory, setScopedHistory] = createGetterSetter abortController.abort()); - const inspectorAdapters = this.getInspectorAdapters(); - let inspectorRequest: RequestResponder | undefined; - if (inspectorAdapters?.requests) { - inspectorRequest = inspectorAdapters.requests.start(requestName, { - id: requestId, - description: requestDescription, - searchSessionId, - }); - } + const requestResponder = this.getInspectorAdapters()?.requests?.start(requestName, { + id: requestId, + description: requestDescription, + searchSessionId, + }); let resp; try { - if (inspectorRequest) { - const requestStats = search.getRequestInspectorStats(searchSource); - inspectorRequest.stats(requestStats); - searchSource.getSearchRequestBody().then((body) => { - if (inspectorRequest) { - inspectorRequest.json(body); - } - }); - } - resp = await searchSource.fetch({ - abortSignal: abortController.signal, - sessionId: searchSessionId, - legacyHitsTotal: false, - }); - if (inspectorRequest) { - const responseStats = search.getResponseInspectorStats(resp, searchSource); - inspectorRequest.stats(responseStats).ok({ json: resp }); - } + resp = await searchSource + .fetch$({ + abortSignal: abortController.signal, + sessionId: searchSessionId, + legacyHitsTotal: false, + requestResponder, + }) + .toPromise(); } catch (error) { - if (inspectorRequest) { - inspectorRequest.error(error); - } if (isSearchSourceAbortError(error)) { throw new DataRequestAbortError(); } diff --git a/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts index 85c5379a63b7f..01959ed08036d 100644 --- a/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts +++ b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts @@ -79,7 +79,7 @@ export class CsvGenerator { searchSource: ISearchSource, scrollSettings: CsvExportSettings['scroll'] ) { - const searchBody = await searchSource.getSearchRequestBody(); + const searchBody = searchSource.getSearchRequestBody(); this.logger.debug(`executing search request`); const searchParams = { params: { From 6970b30f0cbe2f74f3eb7cbbbbe8599bb1426b67 Mon Sep 17 00:00:00 2001 From: Larry Gregory Date: Fri, 9 Apr 2021 11:43:58 -0400 Subject: [PATCH 5/8] Document telemetry fields for stack security features (#96638) --- .../security_usage_collector.ts | 23 ++++++++++++++ .../schema/xpack_plugins.json | 30 +++++++++++++++---- 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/security/server/usage_collector/security_usage_collector.ts b/x-pack/plugins/security/server/usage_collector/security_usage_collector.ts index a59951f5fcfe2..813e23a13ff37 100644 --- a/x-pack/plugins/security/server/usage_collector/security_usage_collector.ts +++ b/x-pack/plugins/security/server/usage_collector/security_usage_collector.ts @@ -53,26 +53,49 @@ export function registerSecurityUsageCollector({ usageCollection, config, licens schema: { auditLoggingEnabled: { type: 'boolean', + _meta: { + description: + 'Indicates if audit logging is both enabled and supported by the current license.', + }, }, loginSelectorEnabled: { type: 'boolean', + _meta: { + description: 'Indicates if the login selector UI is enabled.', + }, }, accessAgreementEnabled: { type: 'boolean', + _meta: { + description: + 'Indicates if the access agreement UI is both enabled and supported by the current license.', + }, }, authProviderCount: { type: 'long', + _meta: { + description: + 'The number of configured auth providers (including disabled auth providers).', + }, }, enabledAuthProviders: { type: 'array', items: { type: 'keyword', + _meta: { + description: + 'The types of enabled auth providers (such as `saml`, `basic`, `pki`, etc).', + }, }, }, httpAuthSchemes: { type: 'array', items: { type: 'keyword', + _meta: { + description: + 'The set of enabled http auth schemes. Used for api-based usage, and when credentials are provided via reverse-proxy.', + }, }, }, }, diff --git a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json index e1e0711c2bb2c..3d302aa12832e 100644 --- a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json +++ b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json @@ -3896,27 +3896,45 @@ "security": { "properties": { "auditLoggingEnabled": { - "type": "boolean" + "type": "boolean", + "_meta": { + "description": "Indicates if audit logging is both enabled and supported by the current license." + } }, "loginSelectorEnabled": { - "type": "boolean" + "type": "boolean", + "_meta": { + "description": "Indicates if the login selector UI is enabled." + } }, "accessAgreementEnabled": { - "type": "boolean" + "type": "boolean", + "_meta": { + "description": "Indicates if the access agreement UI is both enabled and supported by the current license." + } }, "authProviderCount": { - "type": "long" + "type": "long", + "_meta": { + "description": "The number of configured auth providers (including disabled auth providers)." + } }, "enabledAuthProviders": { "type": "array", "items": { - "type": "keyword" + "type": "keyword", + "_meta": { + "description": "The types of enabled auth providers (such as `saml`, `basic`, `pki`, etc)." + } } }, "httpAuthSchemes": { "type": "array", "items": { - "type": "keyword" + "type": "keyword", + "_meta": { + "description": "The set of enabled http auth schemes. Used for api-based usage, and when credentials are provided via reverse-proxy." + } } } } From 1eea903269e350ebb68b0dfc73e21165bdd6999f Mon Sep 17 00:00:00 2001 From: Pierre Gayvallet Date: Fri, 9 Apr 2021 18:27:24 +0200 Subject: [PATCH 6/8] fix config validation (#96502) --- .../config/ensure_valid_configuration.test.ts | 46 +++++++++++++++++-- .../config/ensure_valid_configuration.ts | 25 ++++++---- src/core/server/dev/dev_config.ts | 16 ------- src/core/server/dev/index.ts | 9 ---- src/core/server/server.ts | 2 - 5 files changed, 58 insertions(+), 40 deletions(-) delete mode 100644 src/core/server/dev/dev_config.ts delete mode 100644 src/core/server/dev/index.ts diff --git a/src/core/server/config/ensure_valid_configuration.test.ts b/src/core/server/config/ensure_valid_configuration.test.ts index 474e8dd59b4c4..f1006f93dbc2d 100644 --- a/src/core/server/config/ensure_valid_configuration.test.ts +++ b/src/core/server/config/ensure_valid_configuration.test.ts @@ -16,14 +16,40 @@ describe('ensureValidConfiguration', () => { beforeEach(() => { jest.clearAllMocks(); configService = configServiceMock.create(); - configService.getUsedPaths.mockReturnValue(Promise.resolve(['core', 'elastic'])); + + configService.validate.mockResolvedValue(); + configService.getUsedPaths.mockReturnValue(Promise.resolve([])); }); - it('returns normally when there is no unused keys', async () => { - configService.getUnusedPaths.mockResolvedValue([]); + it('returns normally when there is no unused keys and when the config validates', async () => { await expect(ensureValidConfiguration(configService as any)).resolves.toBeUndefined(); }); + it('throws when config validation fails', async () => { + configService.validate.mockImplementation(() => { + throw new Error('some message'); + }); + + await expect(ensureValidConfiguration(configService as any)).rejects.toMatchInlineSnapshot( + `[Error: some message]` + ); + }); + + it('throws a `CriticalError` with the correct processExitCode value when config validation fails', async () => { + expect.assertions(2); + + configService.validate.mockImplementation(() => { + throw new Error('some message'); + }); + + try { + await ensureValidConfiguration(configService as any); + } catch (e) { + expect(e).toBeInstanceOf(CriticalError); + expect(e.processExitCode).toEqual(78); + } + }); + it('throws when there are some unused keys', async () => { configService.getUnusedPaths.mockResolvedValue(['some.key', 'some.other.key']); @@ -44,4 +70,18 @@ describe('ensureValidConfiguration', () => { expect(e.processExitCode).toEqual(64); } }); + + it('does not throw when all unused keys are included in the ignored paths', async () => { + configService.getUnusedPaths.mockResolvedValue(['dev.someDevKey', 'elastic.apm.enabled']); + + await expect(ensureValidConfiguration(configService as any)).resolves.toBeUndefined(); + }); + + it('throws when only some keys are included in the ignored paths', async () => { + configService.getUnusedPaths.mockResolvedValue(['dev.someDevKey', 'some.key']); + + await expect(ensureValidConfiguration(configService as any)).rejects.toMatchInlineSnapshot( + `[Error: Unknown configuration key(s): "some.key". Check for spelling errors and ensure that expected plugins are installed.]` + ); + }); }); diff --git a/src/core/server/config/ensure_valid_configuration.ts b/src/core/server/config/ensure_valid_configuration.ts index a33625cc0841d..c7a4721b7d2ae 100644 --- a/src/core/server/config/ensure_valid_configuration.ts +++ b/src/core/server/config/ensure_valid_configuration.ts @@ -9,22 +9,27 @@ import { ConfigService } from '@kbn/config'; import { CriticalError } from '../errors'; +const ignoredPaths = ['dev.', 'elastic.apm.']; + +const invalidConfigExitCode = 78; +const legacyInvalidConfigExitCode = 64; + export async function ensureValidConfiguration(configService: ConfigService) { - await configService.validate(); + try { + await configService.validate(); + } catch (e) { + throw new CriticalError(e.message, 'InvalidConfig', invalidConfigExitCode, e); + } - const unusedConfigKeys = await configService.getUnusedPaths(); + const unusedPaths = await configService.getUnusedPaths(); + const unusedConfigKeys = unusedPaths.filter((unusedPath) => { + return !ignoredPaths.some((ignoredPath) => unusedPath.startsWith(ignoredPath)); + }); if (unusedConfigKeys.length > 0) { const message = `Unknown configuration key(s): ${unusedConfigKeys .map((key) => `"${key}"`) .join(', ')}. Check for spelling errors and ensure that expected plugins are installed.`; - throw new InvalidConfigurationError(message); - } -} - -class InvalidConfigurationError extends CriticalError { - constructor(message: string) { - super(message, 'InvalidConfig', 64); - Object.setPrototypeOf(this, InvalidConfigurationError.prototype); + throw new CriticalError(message, 'InvalidConfig', legacyInvalidConfigExitCode); } } diff --git a/src/core/server/dev/dev_config.ts b/src/core/server/dev/dev_config.ts deleted file mode 100644 index 2fec778d85713..0000000000000 --- a/src/core/server/dev/dev_config.ts +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { schema } from '@kbn/config-schema'; - -export const config = { - path: 'dev', - // dev configuration is validated by the dev cli. - // we only need to register the `dev` schema to avoid failing core's config validation - schema: schema.object({}, { unknowns: 'ignore' }), -}; diff --git a/src/core/server/dev/index.ts b/src/core/server/dev/index.ts deleted file mode 100644 index 70257d2a5e6c5..0000000000000 --- a/src/core/server/dev/index.ts +++ /dev/null @@ -1,9 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -export { config } from './dev_config'; diff --git a/src/core/server/server.ts b/src/core/server/server.ts index b34d7fec3dcbf..45d11f9013fed 100644 --- a/src/core/server/server.ts +++ b/src/core/server/server.ts @@ -36,7 +36,6 @@ import { config as cspConfig } from './csp'; import { config as elasticsearchConfig } from './elasticsearch'; import { config as httpConfig } from './http'; import { config as loggingConfig } from './logging'; -import { config as devConfig } from './dev'; import { config as kibanaConfig } from './kibana_config'; import { savedObjectsConfig, savedObjectsMigrationConfig } from './saved_objects'; import { config as uiSettingsConfig } from './ui_settings'; @@ -303,7 +302,6 @@ export class Server { loggingConfig, httpConfig, pluginsConfig, - devConfig, kibanaConfig, savedObjectsConfig, savedObjectsMigrationConfig, From d6fa48b2e8611161cb706a7549ac388689cc07d4 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Fri, 9 Apr 2021 13:09:09 -0500 Subject: [PATCH 7/8] skip copy_to_space_flyout_internal.test.tsx #96708 --- .../components/copy_to_space_flyout_internal.test.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/spaces/public/copy_saved_objects_to_space/components/copy_to_space_flyout_internal.test.tsx b/x-pack/plugins/spaces/public/copy_saved_objects_to_space/components/copy_to_space_flyout_internal.test.tsx index e0326a6c9ff11..4392b71d28b61 100644 --- a/x-pack/plugins/spaces/public/copy_saved_objects_to_space/components/copy_to_space_flyout_internal.test.tsx +++ b/x-pack/plugins/spaces/public/copy_saved_objects_to_space/components/copy_to_space_flyout_internal.test.tsx @@ -112,7 +112,7 @@ const setup = async (opts: SetupOpts = {}) => { return { wrapper, onClose, mockSpacesManager, mockToastNotifications, savedObjectToCopy }; }; -describe('CopyToSpaceFlyout', () => { +describe.skip('CopyToSpaceFlyout', () => { it('waits for spaces to load', async () => { const { wrapper } = await setup({ returnBeforeSpacesLoad: true }); From 51ba5f106dfa3f68d355ed5f45cff9c20e4bdab6 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Fri, 9 Apr 2021 13:12:11 -0500 Subject: [PATCH 8/8] skip flyout test, add linked issue #96708 --- .../components/copy_to_space_flyout_internal.test.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/spaces/public/copy_saved_objects_to_space/components/copy_to_space_flyout_internal.test.tsx b/x-pack/plugins/spaces/public/copy_saved_objects_to_space/components/copy_to_space_flyout_internal.test.tsx index 4392b71d28b61..cb821061b9251 100644 --- a/x-pack/plugins/spaces/public/copy_saved_objects_to_space/components/copy_to_space_flyout_internal.test.tsx +++ b/x-pack/plugins/spaces/public/copy_saved_objects_to_space/components/copy_to_space_flyout_internal.test.tsx @@ -112,6 +112,7 @@ const setup = async (opts: SetupOpts = {}) => { return { wrapper, onClose, mockSpacesManager, mockToastNotifications, savedObjectToCopy }; }; +// flaky https://github.com/elastic/kibana/issues/96708 describe.skip('CopyToSpaceFlyout', () => { it('waits for spaces to load', async () => { const { wrapper } = await setup({ returnBeforeSpacesLoad: true });