diff --git a/x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/types.ts b/x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/types.ts index f962204c6b1b4..5466ba2203714 100644 --- a/x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/types.ts +++ b/x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/types.ts @@ -6,26 +6,35 @@ import * as t from 'io-ts'; +export const RuleTypeSchema = t.keyof({ + query: null, + saved_query: null, + machine_learning: null, +}); +export type RuleType = t.TypeOf; + export const NewRuleSchema = t.intersection([ t.type({ description: t.string, enabled: t.boolean, - filters: t.array(t.unknown), - index: t.array(t.string), interval: t.string, - language: t.string, name: t.string, - query: t.string, risk_score: t.number, severity: t.string, - type: t.union([t.literal('query'), t.literal('saved_query')]), + type: RuleTypeSchema, }), t.partial({ + anomaly_threshold: t.number, created_by: t.string, false_positives: t.array(t.string), + filters: t.array(t.unknown), from: t.string, id: t.string, + index: t.array(t.string), + language: t.string, + machine_learning_job_id: t.string, max_signals: t.number, + query: t.string, references: t.array(t.string), rule_id: t.string, saved_id: t.string, @@ -56,32 +65,34 @@ export const RuleSchema = t.intersection([ description: t.string, enabled: t.boolean, false_positives: t.array(t.string), - filters: t.array(t.unknown), from: t.string, id: t.string, - index: t.array(t.string), interval: t.string, immutable: t.boolean, - language: t.string, name: t.string, max_signals: t.number, - query: t.string, references: t.array(t.string), risk_score: t.number, rule_id: t.string, severity: t.string, tags: t.array(t.string), - type: t.string, + type: RuleTypeSchema, to: t.string, threat: t.array(t.unknown), updated_at: t.string, updated_by: t.string, }), t.partial({ + anomaly_threshold: t.number, + filters: t.array(t.unknown), + index: t.array(t.string), + language: t.string, last_failure_at: t.string, last_failure_message: t.string, meta: MetaRule, + machine_learning_job_id: t.string, output_index: t.string, + query: t.string, saved_id: t.string, status: t.string, status_date: t.string, diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/__mocks__/mock.ts b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/__mocks__/mock.ts index 5627d33818500..011a2614c1af9 100644 --- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/__mocks__/mock.ts +++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/__mocks__/mock.ts @@ -181,6 +181,9 @@ export const mockAboutStepRule = (isNew = false): AboutStepRule => ({ export const mockDefineStepRule = (isNew = false): DefineStepRule => ({ isNew, + ruleType: 'query', + anomalyThreshold: 50, + machineLearningJobId: '', index: ['filebeat-'], queryBar: mockQueryBar, }); diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/anomaly_threshold_slider/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/anomaly_threshold_slider/index.tsx new file mode 100644 index 0000000000000..18970ff935b8d --- /dev/null +++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/anomaly_threshold_slider/index.tsx @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { useCallback } from 'react'; +import { EuiFlexGrid, EuiFlexItem, EuiRange, EuiFormRow } from '@elastic/eui'; + +import { FieldHook } from '../../../../../shared_imports'; + +interface AnomalyThresholdSliderProps { + field: FieldHook; +} +type Event = React.ChangeEvent; +type EventArg = Event | React.MouseEvent; + +export const AnomalyThresholdSlider: React.FC = ({ field }) => { + const threshold = field.value as number; + const onThresholdChange = useCallback( + (event: EventArg) => { + const thresholdValue = Number((event as Event).target.value); + field.setValue(thresholdValue); + }, + [field] + ); + + return ( + + + + + + + + ); +}; diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/helpers.test.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/helpers.test.tsx index 56c9d6da15607..7a3f0105d3d15 100644 --- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/helpers.test.tsx +++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/helpers.test.tsx @@ -38,10 +38,7 @@ setupMock.uiSettings.get.mockImplementation(uiSettingsMock(true)); const mockFilterManager = new FilterManager(setupMock.uiSettings); const mockQueryBar = { - query: { - query: 'test query', - language: 'kuery', - }, + query: 'test query', filters: [ { $state: { @@ -93,10 +90,7 @@ describe('helpers', () => { describe('buildQueryBarDescription', () => { test('returns empty array if no filters, query or savedId exist', () => { const emptyMockQueryBar = { - query: { - query: '', - language: 'kuery', - }, + query: '', filters: [], saved_id: '', }; @@ -113,10 +107,7 @@ describe('helpers', () => { test('returns expected array of ListItems when filters exists, but no indexPatterns passed in', () => { const mockQueryBarWithFilters = { ...mockQueryBar, - query: { - query: '', - language: 'kuery', - }, + query: '', saved_id: '', }; const result: ListItems[] = buildQueryBarDescription({ @@ -135,10 +126,7 @@ describe('helpers', () => { test('returns expected array of ListItems when filters AND indexPatterns exist', () => { const mockQueryBarWithFilters = { ...mockQueryBar, - query: { - query: '', - language: 'kuery', - }, + query: '', saved_id: '', }; const result: ListItems[] = buildQueryBarDescription({ @@ -171,16 +159,13 @@ describe('helpers', () => { savedId: mockQueryBarWithQuery.saved_id, }); expect(result[0].title).toEqual(<>{i18n.QUERY_LABEL} ); - expect(result[0].description).toEqual(<>{mockQueryBarWithQuery.query.query} ); + expect(result[0].description).toEqual(<>{mockQueryBarWithQuery.query} ); }); test('returns expected array of ListItems when "savedId" exists', () => { const mockQueryBarWithSavedId = { ...mockQueryBar, - query: { - query: '', - language: 'kuery', - }, + query: '', filters: [], }; const result: ListItems[] = buildQueryBarDescription({ diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/helpers.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/helpers.tsx index bc454ecb1134a..7b22078c89d1b 100644 --- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/helpers.tsx +++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/helpers.tsx @@ -77,12 +77,12 @@ export const buildQueryBarDescription = ({ }, ]; } - if (!isEmpty(query.query)) { + if (!isEmpty(query)) { items = [ ...items, { title: <>{i18n.QUERY_LABEL} , - description: <>{query.query} , + description: <>{query} , }, ]; } diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/index.tsx index 1d58ef8014899..43b4a5f781b89 100644 --- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/index.tsx +++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/index.tsx @@ -5,7 +5,7 @@ */ import { EuiDescriptionList, EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; -import { isEmpty, chunk, get, pick } from 'lodash/fp'; +import { isEmpty, chunk, get, pick, isNumber } from 'lodash/fp'; import React, { memo, useState } from 'react'; import styled from 'styled-components'; @@ -14,7 +14,6 @@ import { Filter, esFilters, FilterManager, - Query, } from '../../../../../../../../../../src/plugins/data/public'; import { DEFAULT_TIMELINE_TITLE } from '../../../../../components/timeline/translations'; import { useKibana } from '../../../../../lib/kibana'; @@ -133,14 +132,14 @@ export const addFilterStateIfNotThere = (filters: Filter[]): Filter[] => { export const getDescriptionItem = ( field: string, label: string, - value: unknown, + data: unknown, filterManager: FilterManager, indexPatterns?: IIndexPattern ): ListItems[] => { if (field === 'queryBar') { - const filters = addFilterStateIfNotThere(get('queryBar.filters', value) ?? []); - const query = get('queryBar.query', value) as Query; - const savedId = get('queryBar.saved_id', value); + const filters = addFilterStateIfNotThere(get('queryBar.filters', data) ?? []); + const query = get('queryBar.query.query', data); + const savedId = get('queryBar.saved_id', data); return buildQueryBarDescription({ field, filters, @@ -150,31 +149,24 @@ export const getDescriptionItem = ( indexPatterns, }); } else if (field === 'threat') { - const threat: IMitreEnterpriseAttack[] = get(field, value).filter( + const threat: IMitreEnterpriseAttack[] = get(field, data).filter( (singleThreat: IMitreEnterpriseAttack) => singleThreat.tactic.name !== 'none' ); return buildThreatDescription({ label, threat }); } else if (field === 'references') { - const urls: string[] = get(field, value); + const urls: string[] = get(field, data); return buildUrlsDescription(label, urls); } else if (field === 'falsePositives') { - const values: string[] = get(field, value); + const values: string[] = get(field, data); return buildUnorderedListArrayDescription(label, field, values); - } else if (Array.isArray(get(field, value))) { - const values: string[] = get(field, value); + } else if (Array.isArray(get(field, data))) { + const values: string[] = get(field, data); return buildStringArrayDescription(label, field, values); } else if (field === 'severity') { - const val: string = get(field, value); + const val: string = get(field, data); return buildSeverityDescription(label, val); - } else if (field === 'riskScore') { - return [ - { - title: label, - description: get(field, value), - }, - ]; } else if (field === 'timeline') { - const timeline = get(field, value) as FieldValueTimeline; + const timeline = get(field, data) as FieldValueTimeline; return [ { title: label, @@ -182,11 +174,12 @@ export const getDescriptionItem = ( }, ]; } else if (field === 'note') { - const val: string = get(field, value); + const val: string = get(field, data); return buildNoteDescription(label, val); } - const description: string = get(field, value); - if (!isEmpty(description)) { + + const description: string = get(field, data); + if (isNumber(description) || !isEmpty(description)) { return [ { title: label, diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/types.ts b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/types.ts index ab73c52ae9070..bfca6b2068443 100644 --- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/types.ts +++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/description_step/types.ts @@ -9,7 +9,6 @@ import { IIndexPattern, Filter, FilterManager, - Query, } from '../../../../../../../../../../src/plugins/data/public'; import { IMitreEnterpriseAttack } from '../../types'; @@ -22,7 +21,7 @@ export interface BuildQueryBarDescription { field: string; filters: Filter[]; filterManager: FilterManager; - query: Query; + query: string; savedId: string; indexPatterns?: IIndexPattern; } diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/ml_job_select/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/ml_job_select/index.tsx new file mode 100644 index 0000000000000..627fa21cc2f61 --- /dev/null +++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/ml_job_select/index.tsx @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { useCallback } from 'react'; +import { EuiFlexGroup, EuiFlexItem, EuiFormRow, EuiSuperSelect, EuiText } from '@elastic/eui'; + +import { FieldHook, getFieldValidityAndErrorMessage } from '../../../../../shared_imports'; +import { useSiemJobs } from '../../../../../components/ml_popover/hooks/use_siem_jobs'; + +const JobDisplay = ({ title, description }: { title: string; description: string }) => ( + <> + {title} + +

{description}

+
+ +); + +interface MlJobSelectProps { + field: FieldHook; +} + +export const MlJobSelect: React.FC = ({ field }) => { + const jobId = field.value as string; + const { isInvalid, errorMessage } = getFieldValidityAndErrorMessage(field); + const [isLoading, siemJobs] = useSiemJobs(false); + const handleJobChange = useCallback( + (machineLearningJobId: string) => { + field.setValue(machineLearningJobId); + }, + [field] + ); + + const options = siemJobs.map(job => ({ + value: job.id, + inputDisplay: job.id, + dropdownDisplay: , + })); + + return ( + + + + + + + + ); +}; diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/query_bar/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/query_bar/index.tsx index 5886a76182eec..d232c86c19e6f 100644 --- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/query_bar/index.tsx +++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/query_bar/index.tsx @@ -35,7 +35,7 @@ import * as i18n from './translations'; export interface FieldValueQueryBar { filters: Filter[]; query: Query; - saved_id: string | null; + saved_id?: string; } interface QueryBarDefineRuleProps { browserFields: BrowserFields; diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/select_rule_type/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/select_rule_type/index.tsx new file mode 100644 index 0000000000000..b3b35699914f6 --- /dev/null +++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/select_rule_type/index.tsx @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { useCallback } from 'react'; +import { EuiCard, EuiFlexGrid, EuiFlexItem, EuiIcon, EuiFormRow } from '@elastic/eui'; + +import { FieldHook } from '../../../../../shared_imports'; +import { RuleType } from '../../../../../containers/detection_engine/rules/types'; +import * as i18n from './translations'; +import { isMlRule } from '../../helpers'; + +interface SelectRuleTypeProps { + field: FieldHook; +} + +export const SelectRuleType: React.FC = ({ field }) => { + const ruleType = field.value as RuleType; + const setType = useCallback( + (type: RuleType) => { + field.setValue(type); + }, + [field] + ); + const setMl = useCallback(() => setType('machine_learning'), [setType]); + const setQuery = useCallback(() => setType('query'), [setType]); + const license = true; // TODO + + return ( + + + + } + selectable={{ + onClick: setQuery, + isSelected: !isMlRule(ruleType), + }} + /> + + + } + selectable={{ + onClick: setMl, + isSelected: isMlRule(ruleType), + }} + /> + + + + ); +}; diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/select_rule_type/translations.ts b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/select_rule_type/translations.ts new file mode 100644 index 0000000000000..32b860e8f703e --- /dev/null +++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/select_rule_type/translations.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { i18n } from '@kbn/i18n'; + +export const QUERY_TYPE_TITLE = i18n.translate( + 'xpack.siem.detectionEngine.createRule.stepDefineRule.ruleTypeField.queryTypeTitle', + { + defaultMessage: 'Custom query', + } +); + +export const QUERY_TYPE_DESCRIPTION = i18n.translate( + 'xpack.siem.detectionEngine.createRule.stepDefineRule.ruleTypeField.queryTypeDescription', + { + defaultMessage: 'Use KQL or Lucene to detect issues across indices.', + } +); + +export const ML_TYPE_TITLE = i18n.translate( + 'xpack.siem.detectionEngine.createRule.stepDefineRule.ruleTypeField.mlTypeTitle', + { + defaultMessage: 'Machine Learning', + } +); + +export const ML_TYPE_DESCRIPTION = i18n.translate( + 'xpack.siem.detectionEngine.createRule.stepDefineRule.ruleTypeField.mlTypeDescription', + { + defaultMessage: 'Select ML job to detect anomalous activity.', + } +); + +export const ML_TYPE_DISABLED_DESCRIPTION = i18n.translate( + 'xpack.siem.detectionEngine.createRule.stepDefineRule.ruleTypeField.mlTypeDisabledDescription', + { + defaultMessage: 'Access to ML requires a Platinum subscription.', + } +); diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/index.tsx index 2327ac36a5906..6b1a9a828d950 100644 --- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/index.tsx +++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/index.tsx @@ -9,6 +9,7 @@ import { EuiHorizontalRule, EuiFlexGroup, EuiFlexItem, + EuiFormRow, EuiButton, } from '@elastic/eui'; import { isEmpty } from 'lodash/fp'; @@ -20,11 +21,14 @@ import { IIndexPattern } from '../../../../../../../../../../src/plugins/data/pu import { useFetchIndexPatterns } from '../../../../../containers/detection_engine/rules'; import { DEFAULT_INDEX_KEY } from '../../../../../../common/constants'; import { useUiSetting$ } from '../../../../../lib/kibana'; -import { setFieldValue } from '../../helpers'; +import { setFieldValue, isMlRule } from '../../helpers'; import * as RuleI18n from '../../translations'; import { DefineStepRule, RuleStep, RuleStepProps } from '../../types'; import { StepRuleDescription } from '../description_step'; import { QueryBarDefineRule } from '../query_bar'; +import { SelectRuleType } from '../select_rule_type'; +import { AnomalyThresholdSlider } from '../anomaly_threshold_slider'; +import { MlJobSelect } from '../ml_job_select'; import { StepContentWrapper } from '../step_content_wrapper'; import { Field, @@ -33,9 +37,11 @@ import { getUseField, UseField, useForm, + FormSchema, } from '../../../../../shared_imports'; import { schema } from './schema'; import * as i18n from './translations'; +import { filterRuleFieldsForType, RuleFields } from '../../create/helpers'; const CommonUseField = getUseField({ component: Field }); @@ -43,13 +49,16 @@ interface StepDefineRuleProps extends RuleStepProps { defaultValues?: DefineStepRule | null; } -const stepDefineDefaultValue = { +const stepDefineDefaultValue: DefineStepRule = { + anomalyThreshold: 50, index: [], isNew: true, + machineLearningJobId: '', + ruleType: 'query', queryBar: { query: { query: '', language: 'kuery' }, filters: [], - saved_id: null, + saved_id: undefined, }, }; @@ -96,6 +105,7 @@ const StepDefineRuleComponent: FC = ({ }) => { const [openTimelineSearch, setOpenTimelineSearch] = useState(false); const [localUseIndicesConfig, setLocalUseIndicesConfig] = useState(false); + const [localIsMlRule, setIsMlRule] = useState(false); const [indicesConfig] = useUiSetting$(DEFAULT_INDEX_KEY); const [mylocalIndicesConfig, setMyLocalIndicesConfig] = useState( defaultValues != null ? defaultValues.index : indicesConfig ?? [] @@ -112,6 +122,7 @@ const StepDefineRuleComponent: FC = ({ options: { stripEmptyFields: false }, schema, }); + const clearErrors = useCallback(() => form.reset({ resetValues: false }), [form]); const onSubmit = useCallback(async () => { if (setStepData) { @@ -154,64 +165,75 @@ const StepDefineRuleComponent: FC = ({ setOpenTimelineSearch(false); }, []); - return isReadOnlyView && myStepData?.queryBar != null ? ( + return isReadOnlyView ? ( ) : ( <>
- - {i18n.RESET_DEFAULT_INDEX} - - ) : null, - }} - componentProps={{ - idAria: 'detectionEngineStepDefineRuleIndices', - 'data-test-subj': 'detectionEngineStepDefineRuleIndices', - euiFieldProps: { - fullWidth: true, - isDisabled: isLoading, - placeholder: '', - }, - }} - /> - - {i18n.IMPORT_TIMELINE_QUERY} - - ), - }} - component={QueryBarDefineRule} - componentProps={{ - browserFields, - loading: indexPatternLoadingQueryBar, - idAria: 'detectionEngineStepDefineRuleQueryBar', - indexPattern: indexPatternQueryBar, - isDisabled: isLoading, - isLoading: indexPatternLoadingQueryBar, - dataTestSubj: 'detectionEngineStepDefineRuleQueryBar', - openTimelineSearch, - onCloseTimelineSearch: handleCloseTimelineSearch, - }} - /> - - {({ index }) => { + + + <> + + {i18n.RESET_DEFAULT_INDEX} + + ) : null, + }} + componentProps={{ + idAria: 'detectionEngineStepDefineRuleIndices', + 'data-test-subj': 'detectionEngineStepDefineRuleIndices', + euiFieldProps: { + fullWidth: true, + isDisabled: isLoading, + placeholder: '', + }, + }} + /> + + {i18n.IMPORT_TIMELINE_QUERY} + + ), + }} + component={QueryBarDefineRule} + componentProps={{ + browserFields, + loading: indexPatternLoadingQueryBar, + idAria: 'detectionEngineStepDefineRuleQueryBar', + indexPattern: indexPatternQueryBar, + isDisabled: isLoading, + isLoading: indexPatternLoadingQueryBar, + dataTestSubj: 'detectionEngineStepDefineRuleQueryBar', + openTimelineSearch, + onCloseTimelineSearch: handleCloseTimelineSearch, + }} + /> + + + + <> + + + + + + {({ index, ruleType }) => { if (index != null) { if (deepEqual(index, indicesConfig) && !localUseIndicesConfig) { setLocalUseIndicesConfig(true); @@ -223,6 +245,15 @@ const StepDefineRuleComponent: FC = ({ setMyLocalIndicesConfig(index); } } + + if (isMlRule(ruleType) && !localIsMlRule) { + setIsMlRule(true); + clearErrors(); + } else if (!isMlRule(ruleType) && localIsMlRule) { + setIsMlRule(false); + clearErrors(); + } + return null; }} diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/schema.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/schema.tsx index e202ff030cd90..bcfcd4f4ee09d 100644 --- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/schema.tsx +++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/components/step_define_rule/schema.tsx @@ -19,8 +19,7 @@ import { ValidationFunc, } from '../../../../../shared_imports'; import { CUSTOM_QUERY_REQUIRED, INVALID_CUSTOM_QUERY, INDEX_HELPER_TEXT } from './translations'; - -const { emptyField } = fieldValidators; +import { isMlRule } from '../../helpers'; export const schema: FormSchema = { index: { @@ -34,14 +33,25 @@ export const schema: FormSchema = { helpText: {INDEX_HELPER_TEXT}, validations: [ { - validator: emptyField( - i18n.translate( - 'xpack.siem.detectionEngine.createRule.stepDefineRule.outputIndiceNameFieldRequiredError', - { - defaultMessage: 'A minimum of one index pattern is required.', - } - ) - ), + validator: ( + ...args: Parameters + ): ReturnType> | undefined => { + const [{ formData }] = args; + const needsValidation = !isMlRule(formData.ruleType); + + if (!needsValidation) { + return; + } + + return fieldValidators.emptyField( + i18n.translate( + 'xpack.siem.detectionEngine.createRule.stepDefineRule.outputIndiceNameFieldRequiredError', + { + defaultMessage: 'A minimum of one index pattern is required.', + } + ) + )(...args); + }, }, ], }, @@ -57,8 +67,13 @@ export const schema: FormSchema = { validator: ( ...args: Parameters ): ReturnType> | undefined => { - const [{ value, path }] = args; + const [{ value, path, formData }] = args; const { query, filters } = value as FieldValueQueryBar; + const needsValidation = !isMlRule(formData.ruleType); + if (!needsValidation) { + return; + } + return isEmpty(query.query as string) && isEmpty(filters) ? { code: 'ERR_FIELD_MISSING', @@ -72,8 +87,13 @@ export const schema: FormSchema = { validator: ( ...args: Parameters ): ReturnType> | undefined => { - const [{ value, path }] = args; + const [{ value, path, formData }] = args; const { query } = value as FieldValueQueryBar; + const needsValidation = !isMlRule(formData.ruleType); + if (!needsValidation) { + return; + } + if (!isEmpty(query.query as string) && query.language === 'kuery') { try { esKuery.fromKueryExpression(query.query); @@ -85,7 +105,55 @@ export const schema: FormSchema = { }; } } - return undefined; + }, + }, + ], + }, + ruleType: { + label: i18n.translate( + 'xpack.siem.detectionEngine.createRule.stepDefineRule.fieldRuleTypeLabel', + { + defaultMessage: 'Rule type', + } + ), + validations: [], + }, + anomalyThreshold: { + label: i18n.translate( + 'xpack.siem.detectionEngine.createRule.stepDefineRule.fieldAnomalyThresholdLabel', + { + defaultMessage: 'Anomaly score threshold', + } + ), + validations: [], + }, + machineLearningJobId: { + label: i18n.translate( + 'xpack.siem.detectionEngine.createRule.stepDefineRule.fieldMachineLearningJobIdLabel', + { + defaultMessage: 'Machine Learning job', + } + ), + validations: [ + { + validator: ( + ...args: Parameters + ): ReturnType> | undefined => { + const [{ formData }] = args; + const needsValidation = isMlRule(formData.ruleType); + + if (!needsValidation) { + return; + } + + return fieldValidators.emptyField( + i18n.translate( + 'xpack.siem.detectionEngine.createRule.stepDefineRule.machineLearningJobIdRequired', + { + defaultMessage: 'A Machine Learning job is required.', + } + ) + )(...args); }, }, ], diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.test.ts b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.test.ts index dbc5dd9bbe29a..ea6b02924cb3e 100644 --- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.test.ts +++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.test.ts @@ -87,6 +87,7 @@ describe('helpers', () => { query: 'test query', saved_id: 'test123', index: ['filebeat-'], + type: 'saved_query', }; expect(result).toEqual(expected); @@ -106,6 +107,8 @@ describe('helpers', () => { filters: mockQueryBar.filters, query: 'test query', index: ['filebeat-'], + saved_id: '', + type: 'query', }; expect(result).toEqual(expected); @@ -574,12 +577,6 @@ describe('helpers', () => { expect(result.type).toEqual('query'); }); - test('returns NewRule with id set to ruleId if ruleId exists', () => { - const result: NewRule = formatRule(mockDefine, mockAbout, mockSchedule, 'query-with-rule-id'); - - expect(result.id).toEqual('query-with-rule-id'); - }); - test('returns NewRule without id if ruleId does not exist', () => { const result: NewRule = formatRule(mockDefine, mockAbout, mockSchedule); diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.ts b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.ts index 07578e870bf2b..1f3379bf681bb 100644 --- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.ts +++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/helpers.ts @@ -4,10 +4,10 @@ * you may not use this file except in compliance with the Elastic License. */ -import { isEmpty } from 'lodash/fp'; +import { has, isEmpty } from 'lodash/fp'; import moment from 'moment'; -import { NewRule } from '../../../../containers/detection_engine/rules'; +import { NewRule, RuleType } from '../../../../containers/detection_engine/rules'; import { AboutStepRule, @@ -16,8 +16,8 @@ import { DefineStepRuleJson, ScheduleStepRuleJson, AboutStepRuleJson, - FormatRuleType, } from '../types'; +import { isMlRule } from '../helpers'; export const getTimeTypeValue = (time: string): { unit: string; value: number } => { const timeObj = { @@ -39,16 +39,52 @@ export const getTimeTypeValue = (time: string): { unit: string; value: number } return timeObj; }; +export interface RuleFields { + anomalyThreshold: unknown; + machineLearningJobId: unknown; + queryBar: unknown; + index: unknown; + ruleType: unknown; +} +type QueryRuleFields = Omit; +type MlRuleFields = Omit; + +const isMlFields = (fields: QueryRuleFields | MlRuleFields): fields is MlRuleFields => + has('anomalyThreshold', fields); + +export const filterRuleFieldsForType = (fields: T, type: RuleType) => { + if (isMlRule(type)) { + const { index, queryBar, ...mlRuleFields } = fields; + return mlRuleFields; + } else { + const { anomalyThreshold, machineLearningJobId, ...queryRuleFields } = fields; + return queryRuleFields; + } +}; + export const formatDefineStepData = (defineStepData: DefineStepRule): DefineStepRuleJson => { - const { queryBar, isNew, ...rest } = defineStepData; - const { filters, query, saved_id: savedId } = queryBar; - return { - ...rest, - language: query.language, - filters, - query: query.query as string, - ...(savedId != null && savedId !== '' ? { saved_id: savedId } : {}), - }; + const ruleFields = filterRuleFieldsForType(defineStepData, defineStepData.ruleType); + + if (isMlFields(ruleFields)) { + const { anomalyThreshold, machineLearningJobId, isNew, ruleType, ...rest } = ruleFields; + return { + ...rest, + type: ruleType, + anomaly_threshold: anomalyThreshold, + machine_learning_job_id: machineLearningJobId, + }; + } else { + const { queryBar, isNew, ruleType, ...rest } = ruleFields; + return { + ...rest, + type: ruleType, + filters: queryBar?.filters, + language: queryBar?.query?.language, + query: queryBar?.query?.query as string, + saved_id: queryBar?.saved_id, + ...(ruleType === 'query' && queryBar?.saved_id ? { type: 'saved_query' as RuleType } : {}), + }; + } }; export const formatScheduleStepData = (scheduleData: ScheduleStepRule): ScheduleStepRuleJson => { @@ -110,15 +146,9 @@ export const formatAboutStepData = (aboutStepData: AboutStepRule): AboutStepRule export const formatRule = ( defineStepData: DefineStepRule, aboutStepData: AboutStepRule, - scheduleData: ScheduleStepRule, - ruleId?: string -): NewRule => { - const type: FormatRuleType = !isEmpty(defineStepData.queryBar.saved_id) ? 'saved_query' : 'query'; - const persistData = { - type, - ...formatDefineStepData(defineStepData), - ...formatAboutStepData(aboutStepData), - ...formatScheduleStepData(scheduleData), - }; - return ruleId != null ? { id: ruleId, ...persistData } : persistData; -}; + scheduleData: ScheduleStepRule +): NewRule => ({ + ...formatDefineStepData(defineStepData), + ...formatAboutStepData(aboutStepData), + ...formatScheduleStepData(scheduleData), +}); diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/index.tsx index c9f44ab0048f9..67aaabfe70fda 100644 --- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/index.tsx +++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/create/index.tsx @@ -98,7 +98,6 @@ const CreateRulePageComponent: React.FC = () => { const userHasNoPermissions = canUserCRUD != null && hasManageApiKey != null ? !canUserCRUD || !hasManageApiKey : false; - // eslint-disable-next-line react-hooks/rules-of-hooks const setStepData = useCallback( (step: RuleStep, data: unknown, isValid: boolean) => { stepsData.current[step] = { ...stepsData.current[step], data, isValid }; @@ -138,12 +137,10 @@ const CreateRulePageComponent: React.FC = () => { [isStepRuleInReadOnlyView, openAccordionId, stepsData.current, setRule] ); - // eslint-disable-next-line react-hooks/rules-of-hooks const setStepsForm = useCallback((step: RuleStep, form: FormHook) => { stepsForm.current[step] = form; }, []); - // eslint-disable-next-line react-hooks/rules-of-hooks const getAccordionType = useCallback( (accordionId: RuleStep) => { if (accordionId === openAccordionId) { diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/edit/index.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/edit/index.tsx index 5e0e4223e3e27..8618bf9504861 100644 --- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/edit/index.tsx +++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/edit/index.tsx @@ -195,8 +195,8 @@ const EditRulePageComponent: FC = () => { if (invalidForms.length === 0 && activeForm != null) { setTabHasError([]); - setRule( - formatRule( + setRule({ + ...formatRule( (activeFormId === RuleStep.defineRule ? activeForm.data : myDefineRuleForm.data) as DefineStepRule, @@ -205,10 +205,10 @@ const EditRulePageComponent: FC = () => { : myAboutRuleForm.data) as AboutStepRule, (activeFormId === RuleStep.scheduleRule ? activeForm.data - : myScheduleRuleForm.data) as ScheduleStepRule, - ruleId - ) - ); + : myScheduleRuleForm.data) as ScheduleStepRule + ), + ...(ruleId ? { id: ruleId } : {}), + }); } else { setTabHasError(invalidForms); } diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/helpers.test.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/helpers.test.tsx index 0c29bc31cdebc..ee43ae5f1d6e2 100644 --- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/helpers.test.tsx +++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/helpers.test.tsx @@ -32,7 +32,10 @@ describe('rule helpers', () => { }); const defineRuleStepData = { isNew: false, + ruleType: 'saved_query', + anomalyThreshold: 50, index: ['auditbeat-*'], + machineLearningJobId: '', queryBar: { query: { query: 'user.name: root or user.name: admin', @@ -180,6 +183,9 @@ describe('rule helpers', () => { const result: DefineStepRule = getDefineStepsData(mockRule('test-id')); const expected = { isNew: false, + ruleType: 'saved_query', + anomalyThreshold: 50, + machineLearningJobId: '', index: ['auditbeat-*'], queryBar: { query: { @@ -194,7 +200,7 @@ describe('rule helpers', () => { expect(result).toEqual(expected); }); - test('returns with saved_id of null if value does not exist on rule', () => { + test('returns with saved_id of undefined if value does not exist on rule', () => { const mockedRule = { ...mockRule('test-id'), }; @@ -202,6 +208,9 @@ describe('rule helpers', () => { const result: DefineStepRule = getDefineStepsData(mockedRule); const expected = { isNew: false, + ruleType: 'saved_query', + anomalyThreshold: 50, + machineLearningJobId: '', index: ['auditbeat-*'], queryBar: { query: { @@ -209,7 +218,7 @@ describe('rule helpers', () => { language: 'kuery', }, filters: [], - saved_id: null, + saved_id: undefined, }, }; diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/helpers.tsx b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/helpers.tsx index 1fc8a86a476f2..e59ca5e7e14e5 100644 --- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/helpers.tsx +++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/helpers.tsx @@ -10,7 +10,7 @@ import moment from 'moment'; import { useLocation } from 'react-router-dom'; import { Filter } from '../../../../../../../../src/plugins/data/public'; -import { Rule } from '../../../containers/detection_engine/rules'; +import { Rule, RuleType } from '../../../containers/detection_engine/rules'; import { FormData, FormHook, FormSchema } from '../../../shared_imports'; import { AboutStepRule, @@ -43,18 +43,16 @@ export const getStepsData = ({ }; export const getDefineStepsData = (rule: Rule): DefineStepRule => { - const { index, query, language, filters, saved_id: savedId } = rule; - return { isNew: false, - index, + ruleType: rule.type, + anomalyThreshold: rule.anomaly_threshold ?? 50, + machineLearningJobId: rule.machine_learning_job_id ?? '', + index: rule.index ?? [], queryBar: { - query: { - query, - language, - }, - filters: filters as Filter[], - saved_id: savedId ?? null, + query: { query: rule.query ?? '', language: rule.language ?? '' }, + filters: (rule.filters ?? []) as Filter[], + saved_id: rule.saved_id, }, }; }; @@ -195,6 +193,8 @@ export const setFieldValue = ( } }); +export const isMlRule = (ruleType: RuleType) => ruleType === 'machine_learning'; + export const redirectToDetections = ( isSignalIndexExists: boolean | null, isAuthenticated: boolean | null, diff --git a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/types.ts b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/types.ts index aa50626a1231a..447b5dc6325ee 100644 --- a/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/types.ts +++ b/x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/types.ts @@ -5,6 +5,7 @@ */ import { Filter } from '../../../../../../../../src/plugins/data/common'; +import { RuleType } from '../../../containers/detection_engine/rules/types'; import { FieldValueQueryBar } from './components/query_bar'; import { FormData, FormHook } from '../../../shared_imports'; import { FieldValueTimeline } from './components/pick_timeline'; @@ -67,8 +68,11 @@ export interface AboutStepRuleDetails { } export interface DefineStepRule extends StepRuleData { + anomalyThreshold: number; index: string[]; + machineLearningJobId: string; queryBar: FieldValueQueryBar; + ruleType: RuleType; } export interface ScheduleStepRule extends StepRuleData { @@ -79,11 +83,14 @@ export interface ScheduleStepRule extends StepRuleData { } export interface DefineStepRuleJson { - index: string[]; - filters: Filter[]; + anomaly_threshold?: number; + index?: string[]; + filters?: Filter[]; + machine_learning_job_id?: string; saved_id?: string; - query: string; - language: string; + query?: string; + language?: string; + type: RuleType; } export interface AboutStepRuleJson { @@ -112,8 +119,6 @@ export type MyRule = Omit body: typicalPayload(), }); +export const createMlRuleRequest = () => { + const { query, language, index, ...mlParams } = typicalPayload(); + + return requestMock.create({ + method: 'post', + path: DETECTION_ENGINE_RULES_URL, + body: { + ...mlParams, + type: 'machine_learning', + anomaly_threshold: 50, + machine_learning_job_id: 'some-uuid', + }, + }); +}; + export const getSetSignalStatusByIdsRequest = () => requestMock.create({ method: 'post', @@ -349,6 +364,7 @@ export const getResult = (): RuleAlertType => ({ alertTypeId: 'siem.signals', consumer: 'siem', params: { + anomalyThreshold: undefined, description: 'Detecting root and admin users', ruleId: 'rule-1', index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], @@ -357,6 +373,7 @@ export const getResult = (): RuleAlertType => ({ immutable: false, query: 'user.name: root or user.name: admin', language: 'kuery', + machineLearningJobId: undefined, outputIndex: '.siem-signals', timelineId: 'some-timeline-id', timelineTitle: 'some-timeline-title', diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_bulk_route.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_bulk_route.test.ts index 6ad9efebce2dd..2b31d37dddddb 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_bulk_route.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_bulk_route.test.ts @@ -137,7 +137,7 @@ describe('create_rules_bulk', () => { const result = server.validate(request); expect(result.badRequest).toHaveBeenCalledWith( - '"value" at position 0 fails because [child "type" fails because ["type" must be one of [query, saved_query]]]' + '"value" at position 0 fails because [child "type" fails because ["type" must be one of [query, saved_query, machine_learning]]]' ); }); }); diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts index d727bbb953d2a..b819bc6919274 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts @@ -56,12 +56,14 @@ export const createRulesBulkRoute = (router: IRouter) => { .filter(rule => rule.rule_id == null || !dupes.includes(rule.rule_id)) .map(async payloadRule => { const { + anomaly_threshold: anomalyThreshold, description, enabled, false_positives: falsePositives, from, query, language, + machine_learning_job_id: machineLearningJobId, output_index: outputIndex, saved_id: savedId, meta, @@ -107,6 +109,7 @@ export const createRulesBulkRoute = (router: IRouter) => { const createdRule = await createRules({ alertsClient, actionsClient, + anomalyThreshold, description, enabled, falsePositives, @@ -114,6 +117,7 @@ export const createRulesBulkRoute = (router: IRouter) => { immutable: false, query, language, + machineLearningJobId, outputIndex: finalIndex, savedId, timelineId, diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_route.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_route.test.ts index d019668e2a8d1..976f371c6b1a6 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_route.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_route.test.ts @@ -14,6 +14,7 @@ import { getNonEmptyIndex, getEmptyIndex, getFindResultWithSingleHit, + createMlRuleRequest, } from '../__mocks__/request_responses'; import { requestContextMock, serverMock, requestMock } from '../__mocks__'; import { createRulesRoute } from './create_rules_route'; @@ -48,6 +49,13 @@ describe('create_rules', () => { }); }); + describe('creating an ML Rule', () => { + it('is successful', async () => { + const response = await server.inject(createMlRuleRequest(), context); + expect(response.status).toEqual(200); + }); + }); + describe('unhappy paths', () => { test('it returns a 400 if the index does not exist', async () => { clients.clusterClient.callAsCurrentUser.mockResolvedValue(getEmptyIndex()); @@ -111,7 +119,7 @@ describe('create_rules', () => { const result = server.validate(request); expect(result.badRequest).toHaveBeenCalledWith( - 'child "type" fails because ["type" must be one of [query, saved_query]]' + 'child "type" fails because ["type" must be one of [query, saved_query, machine_learning]]' ); }); }); diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_route.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_route.ts index fcfcee99f369e..42bade1ba0855 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_route.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_route.ts @@ -31,6 +31,7 @@ export const createRulesRoute = (router: IRouter): void => { }, async (context, request, response) => { const { + anomaly_threshold: anomalyThreshold, description, enabled, false_positives: falsePositives, @@ -42,6 +43,7 @@ export const createRulesRoute = (router: IRouter): void => { timeline_id: timelineId, timeline_title: timelineTitle, meta, + machine_learning_job_id: machineLearningJobId, filters, rule_id: ruleId, index, @@ -93,6 +95,7 @@ export const createRulesRoute = (router: IRouter): void => { const createdRule = await createRules({ alertsClient, actionsClient, + anomalyThreshold, description, enabled, falsePositives, @@ -105,6 +108,7 @@ export const createRulesRoute = (router: IRouter): void => { timelineId, timelineTitle, meta, + machineLearningJobId, filters, ruleId: ruleId ?? uuid.v4(), index, diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/import_rules_route.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/import_rules_route.ts index ec4e707f46e50..d92ef316aef0c 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/import_rules_route.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/import_rules_route.ts @@ -111,6 +111,7 @@ export const importRulesRoute = (router: IRouter, config: LegacyServices['config return null; } const { + anomaly_threshold: anomalyThreshold, description, enabled, false_positives: falsePositives, @@ -118,6 +119,7 @@ export const importRulesRoute = (router: IRouter, config: LegacyServices['config immutable, query, language, + machine_learning_job_id: machineLearningJobId, output_index: outputIndex, saved_id: savedId, meta, @@ -139,6 +141,7 @@ export const importRulesRoute = (router: IRouter, config: LegacyServices['config timeline_title: timelineTitle, version, } = parsedRule; + try { const signalsIndex = siemClient.signalsIndex; const indexExists = await getIndexExists( @@ -159,6 +162,7 @@ export const importRulesRoute = (router: IRouter, config: LegacyServices['config await createRules({ alertsClient, actionsClient, + anomalyThreshold, description, enabled, falsePositives, @@ -166,6 +170,7 @@ export const importRulesRoute = (router: IRouter, config: LegacyServices['config immutable, query, language, + machineLearningJobId, outputIndex: signalsIndex, savedId, timelineId, diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.test.ts index 19bcd2e7f0596..967fd46f7e3da 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.test.ts @@ -89,7 +89,7 @@ describe('patch_rules_bulk', () => { const result = server.validate(request); expect(result.badRequest).toHaveBeenCalledWith( - '"value" at position 0 fails because [child "type" fails because ["type" must be one of [query, saved_query]]]' + '"value" at position 0 fails because [child "type" fails because ["type" must be one of [query, saved_query, machine_learning]]]' ); }); }); diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/patch_rules_route.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/patch_rules_route.test.ts index 1658de77e3390..0c2ca882a5590 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/patch_rules_route.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/patch_rules_route.test.ts @@ -112,7 +112,7 @@ describe('patch_rules', () => { const result = server.validate(request); expect(result.badRequest).toHaveBeenCalledWith( - 'child "type" fails because ["type" must be one of [query, saved_query]]' + 'child "type" fails because ["type" must be one of [query, saved_query, machine_learning]]' ); }); }); diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/update_rules_bulk_route.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/update_rules_bulk_route.test.ts index 7a9159ecc852b..46639e1fe3380 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/update_rules_bulk_route.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/update_rules_bulk_route.test.ts @@ -110,7 +110,7 @@ describe('update_rules_bulk', () => { const result = server.validate(request); expect(result.badRequest).toHaveBeenCalledWith( - '"value" at position 0 fails because [child "type" fails because ["type" must be one of [query, saved_query]]]' + '"value" at position 0 fails because [child "type" fails because ["type" must be one of [query, saved_query, machine_learning]]]' ); }); }); diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/update_rules_bulk_route.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/update_rules_bulk_route.ts index 777b9f3cc7a9d..859935d851126 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/update_rules_bulk_route.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/update_rules_bulk_route.ts @@ -47,12 +47,14 @@ export const updateRulesBulkRoute = (router: IRouter) => { const rules = await Promise.all( request.body.map(async payloadRule => { const { + anomaly_threshold: anomalyThreshold, description, enabled, false_positives: falsePositives, from, query, language, + machine_learning_job_id: machineLearningJobId, output_index: outputIndex, saved_id: savedId, timeline_id: timelineId, @@ -81,6 +83,7 @@ export const updateRulesBulkRoute = (router: IRouter) => { const rule = await updateRules({ alertsClient, actionsClient, + anomalyThreshold, description, enabled, immutable: false, @@ -88,6 +91,7 @@ export const updateRulesBulkRoute = (router: IRouter) => { from, query, language, + machineLearningJobId, outputIndex: finalIndex, savedId, savedObjectsClient, diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/update_rules_route.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/update_rules_route.test.ts index 6ef508b817713..a6da8cd56ec17 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/update_rules_route.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/update_rules_route.test.ts @@ -115,7 +115,7 @@ describe('update_rules', () => { const result = await server.validate(request); expect(result.badRequest).toHaveBeenCalledWith( - 'child "type" fails because ["type" must be one of [query, saved_query]]' + 'child "type" fails because ["type" must be one of [query, saved_query, machine_learning]]' ); }); }); diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/update_rules_route.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/update_rules_route.ts index 1393de8c725cb..a9982a9896633 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/update_rules_route.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/update_rules_route.ts @@ -30,12 +30,14 @@ export const updateRulesRoute = (router: IRouter) => { }, async (context, request, response) => { const { + anomaly_threshold: anomalyThreshold, description, enabled, false_positives: falsePositives, from, query, language, + machine_learning_job_id: machineLearningJobId, output_index: outputIndex, saved_id: savedId, timeline_id: timelineId, @@ -77,6 +79,7 @@ export const updateRulesRoute = (router: IRouter) => { const rule = await updateRules({ alertsClient, actionsClient, + anomalyThreshold, description, enabled, falsePositives, @@ -84,6 +87,7 @@ export const updateRulesRoute = (router: IRouter) => { immutable: false, query, language, + machineLearningJobId, outputIndex: finalIndex, savedId, savedObjectsClient, diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.test.ts index 70fcbb2c163ca..3243ccb14f89c 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.test.ts @@ -36,7 +36,7 @@ describe('utils', () => { test('should work with a full data set', () => { const fullRule = getResult(); const rule = transformAlertToRule(fullRule); - const expected: OutputRuleAlertRest = { + const expected: Partial = { created_by: 'elastic', created_at: '2019-12-13T16:40:33.400Z', updated_at: '2019-12-13T16:40:33.400Z', @@ -358,7 +358,7 @@ describe('utils', () => { const fullRule = getResult(); fullRule.enabled = false; const ruleWithEnabledFalse = transformAlertToRule(fullRule); - const expected: OutputRuleAlertRest = { + const expected: Partial = { created_by: 'elastic', created_at: '2019-12-13T16:40:33.400Z', updated_at: '2019-12-13T16:40:33.400Z', @@ -424,7 +424,7 @@ describe('utils', () => { const fullRule = getResult(); fullRule.params.immutable = false; const ruleWithEnabledFalse = transformAlertToRule(fullRule); - const expected: OutputRuleAlertRest = { + const expected: Partial = { created_by: 'elastic', created_at: '2019-12-13T16:40:33.400Z', updated_at: '2019-12-13T16:40:33.400Z', @@ -490,7 +490,7 @@ describe('utils', () => { const fullRule = getResult(); fullRule.tags = ['tag 1', 'tag 2', `${INTERNAL_IDENTIFIER}_some_other_value`]; const rule = transformAlertToRule(fullRule); - const expected: OutputRuleAlertRest = { + const expected: Partial = { created_at: '2019-12-13T16:40:33.400Z', updated_at: '2019-12-13T16:40:33.400Z', created_by: 'elastic', @@ -551,6 +551,22 @@ describe('utils', () => { }; expect(rule).toEqual(expected); }); + + it('transforms ML Rule fields', () => { + const mlRule = getResult(); + mlRule.params.anomalyThreshold = 55; + mlRule.params.machineLearningJobId = 'some_job_id'; + mlRule.params.type = 'machine_learning'; + + const rule = transformAlertToRule(mlRule); + expect(rule).toEqual( + expect.objectContaining({ + anomaly_threshold: 55, + machine_learning_job_id: 'some_job_id', + type: 'machine_learning', + }) + ); + }); }); describe('getIdError', () => { @@ -640,7 +656,7 @@ describe('utils', () => { total: 0, data: [getResult()], }); - const expected: OutputRuleAlertRest = { + const expected: Partial = { created_by: 'elastic', created_at: '2019-12-13T16:40:33.400Z', updated_at: '2019-12-13T16:40:33.400Z', @@ -722,7 +738,7 @@ describe('utils', () => { describe('transform', () => { test('outputs 200 if the data is of type siem alert', () => { const output = transform(getResult()); - const expected: OutputRuleAlertRest = { + const expected: Partial = { created_by: 'elastic', created_at: '2019-12-13T16:40:33.400Z', updated_at: '2019-12-13T16:40:33.400Z', @@ -895,7 +911,7 @@ describe('utils', () => { describe('transformOrBulkError', () => { test('outputs 200 if the data is of type siem alert', () => { const output = transformOrBulkError('rule-1', getResult()); - const expected: OutputRuleAlertRest = { + const expected: Partial = { created_by: 'elastic', created_at: '2019-12-13T16:40:33.400Z', updated_at: '2019-12-13T16:40:33.400Z', diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.ts index ecf669b0106c3..abd8dd7e87f03 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.ts @@ -106,6 +106,7 @@ export const transformAlertToRule = ( created_by: alert.createdBy, description: alert.params.description, enabled: alert.enabled, + anomaly_threshold: alert.params.anomalyThreshold, false_positives: alert.params.falsePositives, filters: alert.params.filters, from: alert.params.from, @@ -117,6 +118,7 @@ export const transformAlertToRule = ( language: alert.params.language, output_index: alert.params.outputIndex, max_signals: alert.params.maxSignals, + machine_learning_job_id: alert.params.machineLearningJobId, risk_score: alert.params.riskScore, name: alert.name, query: alert.params.query, diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/add_prepackaged_rules_schema.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/add_prepackaged_rules_schema.ts index 974ddcf35eeb4..ec0a8e7871b5b 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/add_prepackaged_rules_schema.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/add_prepackaged_rules_schema.ts @@ -34,6 +34,8 @@ import { references, note, version, + anomaly_threshold, + machine_learning_job_id, } from './schemas'; /* eslint-enable @typescript-eslint/camelcase */ @@ -49,6 +51,11 @@ import { DEFAULT_MAX_SIGNALS } from '../../../../../common/constants'; * - index is a required field that must exist */ export const addPrepackagedRulesSchema = Joi.object({ + anomaly_threshold: anomaly_threshold.when('type', { + is: 'machine_learning', + then: Joi.required(), + otherwise: Joi.forbidden(), + }), description: description.required(), enabled: enabled.default(false), false_positives: false_positives.default([]), @@ -61,8 +68,21 @@ export const addPrepackagedRulesSchema = Joi.object({ .valid(true), index: index.required(), interval: interval.default('5m'), - query: query.allow('').default(''), - language: language.default('kuery'), + query: query.when('type', { + is: 'machine_learning', + then: Joi.forbidden(), + otherwise: query.allow('').default(''), + }), + language: language.when('type', { + is: 'machine_learning', + then: Joi.forbidden(), + otherwise: language.default('kuery'), + }), + machine_learning_job_id: machine_learning_job_id.when('type', { + is: 'machine_learning', + then: Joi.required(), + otherwise: Joi.forbidden(), + }), saved_id: saved_id.when('type', { is: 'saved_query', then: Joi.required(), diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/create_rules_schema.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/create_rules_schema.ts index c9b380d3c67e1..e86963fd4594c 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/create_rules_schema.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/create_rules_schema.ts @@ -8,6 +8,7 @@ import Joi from 'joi'; /* eslint-disable @typescript-eslint/camelcase */ import { + anomaly_threshold, enabled, description, false_positives, @@ -34,12 +35,18 @@ import { references, note, version, + machine_learning_job_id, } from './schemas'; /* eslint-enable @typescript-eslint/camelcase */ import { DEFAULT_MAX_SIGNALS } from '../../../../../common/constants'; export const createRulesSchema = Joi.object({ + anomaly_threshold: anomaly_threshold.when('type', { + is: 'machine_learning', + then: Joi.required(), + otherwise: Joi.forbidden(), + }), description: description.required(), enabled: enabled.default(true), false_positives: false_positives.default([]), @@ -48,8 +55,16 @@ export const createRulesSchema = Joi.object({ rule_id, index, interval: interval.default('5m'), - query: query.allow('').default(''), - language: language.default('kuery'), + query: query.when('type', { + is: 'machine_learning', + then: Joi.forbidden(), + otherwise: query.allow('').default(''), + }), + language: language.when('type', { + is: 'machine_learning', + then: Joi.forbidden(), + otherwise: language.default('kuery'), + }), output_index, saved_id: saved_id.when('type', { is: 'saved_query', @@ -59,6 +74,11 @@ export const createRulesSchema = Joi.object({ timeline_id, timeline_title, meta, + machine_learning_job_id: machine_learning_job_id.when('type', { + is: 'machine_learning', + then: Joi.required(), + otherwise: Joi.forbidden(), + }), risk_score: risk_score.required(), max_signals: max_signals.default(DEFAULT_MAX_SIGNALS), name: name.required(), diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/import_rules_schema.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/import_rules_schema.ts index bd12872c4dc72..92718b7ae71ba 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/import_rules_schema.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/import_rules_schema.ts @@ -40,6 +40,8 @@ import { references, note, version, + anomaly_threshold, + machine_learning_job_id, } from './schemas'; /* eslint-enable @typescript-eslint/camelcase */ @@ -55,6 +57,11 @@ import { DEFAULT_MAX_SIGNALS } from '../../../../../common/constants'; * - updated_by is optional (but ignored in the import code) */ export const importRulesSchema = Joi.object({ + anomaly_threshold: anomaly_threshold.when('type', { + is: 'machine_learning', + then: Joi.required(), + otherwise: Joi.forbidden(), + }), id, description: description.required(), enabled: enabled.default(true), @@ -65,9 +72,22 @@ export const importRulesSchema = Joi.object({ immutable: immutable.default(false).valid(false), index, interval: interval.default('5m'), - query: query.allow('').default(''), - language: language.default('kuery'), + query: query.when('type', { + is: 'machine_learning', + then: Joi.forbidden(), + otherwise: query.allow('').default(''), + }), + language: language.when('type', { + is: 'machine_learning', + then: Joi.forbidden(), + otherwise: language.default('kuery'), + }), output_index, + machine_learning_job_id: machine_learning_job_id.when('type', { + is: 'machine_learning', + then: Joi.required(), + otherwise: Joi.forbidden(), + }), saved_id: saved_id.when('type', { is: 'saved_query', then: Joi.required(), diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/patch_rules_schema.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/patch_rules_schema.ts index 4d1b73fb69e5b..4496a808f6869 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/patch_rules_schema.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/patch_rules_schema.ts @@ -35,10 +35,13 @@ import { note, id, version, + anomaly_threshold, + machine_learning_job_id, } from './schemas'; /* eslint-enable @typescript-eslint/camelcase */ export const patchRulesSchema = Joi.object({ + anomaly_threshold, description, enabled, false_positives, @@ -50,6 +53,7 @@ export const patchRulesSchema = Joi.object({ interval, query: query.allow(''), language, + machine_learning_job_id, output_index, saved_id, timeline_id, diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/__mocks__/utils.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/__mocks__/utils.ts index 05b85ffab7263..dd88bd80d5787 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/__mocks__/utils.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/__mocks__/utils.ts @@ -67,6 +67,18 @@ export const getBaseResponsePayload = (anchorDate: string = ANCHOR_DATE): RulesS export const getRulesBulkPayload = (): RulesBulkSchema => [getBaseResponsePayload()]; +export const getMlRuleResponsePayload = (anchorDate: string = ANCHOR_DATE): RulesSchema => { + const basePayload = getBaseResponsePayload(anchorDate); + const { filters, index, query, language, ...rest } = basePayload; + + return { + ...rest, + type: 'machine_learning', + anomaly_threshold: 59, + machine_learning_job_id: 'some_machine_learning_job_id', + }; +}; + export const getErrorPayload = ( id: string = '819eded6-e9c8-445b-a647-519aea39e063' ): ErrorSchema => ({ diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/check_type_dependents.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/check_type_dependents.test.ts index fc1c019ff97b5..1a5ee793a25da 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/check_type_dependents.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/check_type_dependents.test.ts @@ -12,8 +12,15 @@ import { getDependents, addSavedId, addTimelineTitle, + addQueryFields, + addMlFields, } from './check_type_dependents'; -import { foldLeftRight, getBaseResponsePayload, getPaths } from './__mocks__/utils'; +import { + foldLeftRight, + getBaseResponsePayload, + getPaths, + getMlRuleResponsePayload, +} from './__mocks__/utils'; import { left } from 'fp-ts/lib/Either'; import { exactCheck } from './exact_check'; import { RulesSchema } from './rules_schema'; @@ -375,6 +382,34 @@ describe('check_type_dependents', () => { ]); expect(message.schema).toEqual({}); }); + + test('it validates an ML rule response', () => { + const payload = getMlRuleResponsePayload(); + + const dependents = getDependents(payload); + const decoded = dependents.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + const expected = getMlRuleResponsePayload(); + + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(expected); + }); + + test('it rejects a response with both ML and query properties', () => { + const payload = { + ...getBaseResponsePayload(), + ...getMlRuleResponsePayload(), + }; + + const dependents = getDependents(payload); + const decoded = dependents.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + + expect(getPaths(left(message.errors))).toEqual(['invalid keys "query,language"']); + expect(message.schema).toEqual({}); + }); }); describe('addSavedId', () => { @@ -402,4 +437,35 @@ describe('check_type_dependents', () => { expect(array.length).toEqual(2); }); }); + + describe('addQueryFields', () => { + test('should return empty array if type is not "query"', () => { + const fields = addQueryFields({ type: 'machine_learning' }); + const expected: t.Mixed[] = []; + expect(fields).toEqual(expected); + }); + + test('should return two fields for a rule of type "query"', () => { + const fields = addQueryFields({ type: 'query' }); + expect(fields.length).toEqual(2); + }); + + test('should return two fields for a rule of type "saved_query"', () => { + const fields = addQueryFields({ type: 'saved_query' }); + expect(fields.length).toEqual(2); + }); + }); + + describe('addMlFields', () => { + test('should return empty array if type is not "machine_learning"', () => { + const fields = addMlFields({ type: 'query' }); + const expected: t.Mixed[] = []; + expect(fields).toEqual(expected); + }); + + test('should return two fields for a rule of type "machine_learning"', () => { + const fields = addMlFields({ type: 'machine_learning' }); + expect(fields.length).toEqual(2); + }); + }); }); diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/check_type_dependents.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/check_type_dependents.ts index 09142c8568b2d..b5a01e3e5c6df 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/check_type_dependents.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/check_type_dependents.ts @@ -35,12 +35,38 @@ export const addTimelineTitle = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mi } }; +export const addQueryFields = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mixed[] => { + if (typeAndTimelineOnly.type === 'query' || typeAndTimelineOnly.type === 'saved_query') { + return [ + t.exact(t.type({ query: dependentRulesSchema.props.query })), + t.exact(t.type({ language: dependentRulesSchema.props.language })), + ]; + } else { + return []; + } +}; + +export const addMlFields = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mixed[] => { + if (typeAndTimelineOnly.type === 'machine_learning') { + return [ + t.exact(t.type({ anomaly_threshold: dependentRulesSchema.props.anomaly_threshold })), + t.exact( + t.type({ machine_learning_job_id: dependentRulesSchema.props.machine_learning_job_id }) + ), + ]; + } else { + return []; + } +}; + export const getDependents = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mixed => { const dependents: t.Mixed[] = [ t.exact(requiredRulesSchema), t.exact(partialRulesSchema), ...addSavedId(typeAndTimelineOnly), ...addTimelineTitle(typeAndTimelineOnly), + ...addQueryFields(typeAndTimelineOnly), + ...addMlFields(typeAndTimelineOnly), ]; if (dependents.length > 1) { diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/rules_schema.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/rules_schema.ts index 945b5651be066..28b588a86aeb0 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/rules_schema.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/rules_schema.ts @@ -11,6 +11,7 @@ import { Either } from 'fp-ts/lib/Either'; import { checkTypeDependents } from './check_type_dependents'; import { + anomaly_threshold, description, enabled, false_positives, @@ -24,6 +25,7 @@ import { name, output_index, max_signals, + machine_learning_job_id, query, references, severity, @@ -65,12 +67,10 @@ export const requiredRulesSchema = t.type({ immutable, interval, rule_id, - language, output_index, max_signals, risk_score, name, - query, references, severity, updated_by, @@ -91,12 +91,20 @@ export type RequiredRulesSchema = t.TypeOf; * check_type_dependents file for whichever REST flow it is going through. */ export const dependentRulesSchema = t.partial({ + // query fields + language, + query, + // when type = saved_query, saved_is is required saved_id, // These two are required together or not at all. timeline_id, timeline_title, + + // ML fields + anomaly_threshold, + machine_learning_job_id, }); /** diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/schemas.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/schemas.ts index 16f6c0fd6b8b4..072e3f5beefe2 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/schemas.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/response/schemas.ts @@ -45,6 +45,8 @@ export const output_index = t.string; export const saved_id = t.string; export const timeline_id = t.string; export const timeline_title = t.string; +export const anomaly_threshold = PositiveInteger; +export const machine_learning_job_id = t.string; /** * Note that this is a plain unknown object because we allow the UI @@ -64,7 +66,7 @@ export const job_status = t.keyof({ succeeded: null, failed: null, 'going to run // TODO: Create a regular expression type or custom date math part type here export const to = t.string; -export const type = t.keyof({ query: null, saved_query: null }); +export const type = t.keyof({ machine_learning: null, query: null, saved_query: null }); export const queryFilter = t.string; export const references = t.array(t.string); export const per_page = PositiveInteger; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/schemas.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/schemas.ts index 2ba9ec7f83253..ad7050e8dd65c 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/schemas.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/schemas.ts @@ -7,6 +7,10 @@ import Joi from 'joi'; /* eslint-disable @typescript-eslint/camelcase */ +export const anomaly_threshold = Joi.number() + .integer() + .greater(-1) + .less(101); export const description = Joi.string(); export const enabled = Joi.boolean(); export const exclude_export_details = Joi.boolean(); @@ -48,7 +52,8 @@ export const risk_score = Joi.number() export const severity = Joi.string().valid('low', 'medium', 'high', 'critical'); export const status = Joi.string().valid('open', 'closed'); export const to = Joi.string(); -export const type = Joi.string().valid('query', 'saved_query'); +export const type = Joi.string().valid('query', 'saved_query', 'machine_learning'); +export const machine_learning_job_id = Joi.string(); export const queryFilter = Joi.string(); export const references = Joi.array() .items(Joi.string()) diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/update_rules_schema.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/update_rules_schema.ts index a72105142d287..f7a53385200df 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/update_rules_schema.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/update_rules_schema.ts @@ -35,6 +35,8 @@ import { id, note, version, + anomaly_threshold, + machine_learning_job_id, } from './schemas'; /* eslint-enable @typescript-eslint/camelcase */ @@ -48,6 +50,11 @@ import { DEFAULT_MAX_SIGNALS } from '../../../../../common/constants'; * - id is on here because you can pass in an id to update using it instead of rule_id. */ export const updateRulesSchema = Joi.object({ + anomaly_threshold: anomaly_threshold.when('type', { + is: 'machine_learning', + then: Joi.required(), + otherwise: Joi.forbidden(), + }), description: description.required(), enabled: enabled.default(true), id, @@ -57,8 +64,21 @@ export const updateRulesSchema = Joi.object({ rule_id, index, interval: interval.default('5m'), - query: query.allow('').default(''), - language: language.default('kuery'), + query: query.when('type', { + is: 'machine_learning', + then: Joi.forbidden(), + otherwise: query.allow('').default(''), + }), + language: language.when('type', { + is: 'machine_learning', + then: Joi.forbidden(), + otherwise: language.default('kuery'), + }), + machine_learning_job_id: machine_learning_job_id.when('type', { + is: 'machine_learning', + then: Joi.required(), + otherwise: Joi.forbidden(), + }), output_index, saved_id: saved_id.when('type', { is: 'saved_query', diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/create_rules.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/create_rules.ts index ea87950a59b78..1b4c06fb5d828 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/create_rules.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/create_rules.ts @@ -12,6 +12,7 @@ import { addTags } from './add_tags'; export const createRules = ({ alertsClient, actionsClient, // TODO: Use this actionsClient once we have actions such as email, etc... + anomalyThreshold, description, enabled, falsePositives, @@ -22,6 +23,7 @@ export const createRules = ({ timelineId, timelineTitle, meta, + machineLearningJobId, filters, ruleId, immutable, @@ -47,6 +49,7 @@ export const createRules = ({ alertTypeId: SIGNALS_ID, consumer: APP_ID, params: { + anomalyThreshold, description, ruleId, index, @@ -60,6 +63,7 @@ export const createRules = ({ timelineId, timelineTitle, meta, + machineLearningJobId, filters, maxSignals, riskScore, diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/install_prepacked_rules.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/install_prepacked_rules.ts index 3b5ef57d3dcb6..dc71ae3678f2e 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/install_prepacked_rules.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/install_prepacked_rules.ts @@ -18,6 +18,7 @@ export const installPrepackagedRules = ( ): Array> => rules.reduce>>((acc, rule) => { const { + anomaly_threshold: anomalyThreshold, description, enabled, false_positives: falsePositives, @@ -25,6 +26,7 @@ export const installPrepackagedRules = ( immutable, query, language, + machine_learning_job_id: machineLearningJobId, saved_id: savedId, timeline_id: timelineId, timeline_title: timelineTitle, @@ -50,6 +52,7 @@ export const installPrepackagedRules = ( createRules({ alertsClient, actionsClient, + anomalyThreshold, description, enabled, falsePositives, @@ -57,6 +60,7 @@ export const installPrepackagedRules = ( immutable, query, language, + machineLearningJobId, outputIndex, savedId, timelineId, diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/__mocks__/es_results.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/__mocks__/es_results.ts index 922651edc4082..010f6b2ee98ff 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/__mocks__/es_results.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/__mocks__/es_results.ts @@ -29,6 +29,8 @@ export const sampleRuleAlertParams = ( riskScore: riskScore ? riskScore : 50, maxSignals: maxSignals ? maxSignals : 10000, note: '', + anomalyThreshold: undefined, + machineLearningJobId: undefined, filters: undefined, savedId: undefined, timelineId: undefined, diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/build_rule.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/build_rule.ts index 9baf6a55b7f48..a9ccda2efe99c 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/build_rule.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/build_rule.ts @@ -65,5 +65,7 @@ export const buildRule = ({ version: ruleParams.version, created_at: createdAt, updated_at: updatedAt, + machine_learning_job_id: ruleParams.machineLearningJobId, + anomaly_threshold: ruleParams.anomalyThreshold, }); }; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/bulk_create_ml_signals.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/bulk_create_ml_signals.test.ts new file mode 100644 index 0000000000000..d9fb9d4bbabde --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/bulk_create_ml_signals.test.ts @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { transformAnomalyFieldsToEcs } from './bulk_create_ml_signals'; + +const buildMockAnomaly = () => ({ + job_id: 'rare_process_by_host_linux_ecs', + result_type: 'record', + probability: 0.03406145177566593, + multi_bucket_impact: -0.0, + record_score: 10.86784984522809, + initial_record_score: 10.86784984522809, + bucket_span: 900, + detector_index: 0, + is_interim: false, + timestamp: 1584482400000, + by_field_name: 'process.name', + by_field_value: 'gzip', + partition_field_name: 'host.name', + partition_field_value: 'rock01', + function: 'rare', + function_description: 'rare', + typical: [0.03406145177566593], + actual: [1.0], + influencers: [ + { + influencer_field_name: 'user.name', + influencer_field_values: ['root'], + }, + { + influencer_field_name: 'process.pid', + influencer_field_values: ['123'], + }, + { + influencer_field_name: 'host.name', + influencer_field_values: ['rock01'], + }, + ], + 'process.name': ['gzip'], + 'process.pid': ['123'], + 'user.name': ['root'], + 'host.name': ['rock01'], +}); + +describe('transformAnomalyFieldsToEcs', () => { + it('adds a @timestamp field based on timestamp', () => { + const anomaly = buildMockAnomaly(); + const result = transformAnomalyFieldsToEcs(anomaly); + const expectedTime = '2020-03-17T22:00:00.000Z'; + + expect(result['@timestamp']).toEqual(expectedTime); + }); + + it('deletes dotted influencer fields', () => { + const anomaly = buildMockAnomaly(); + const result = transformAnomalyFieldsToEcs(anomaly); + + const ecsKeys = Object.keys(result); + expect(ecsKeys).not.toContain('user.name'); + expect(ecsKeys).not.toContain('process.pid'); + expect(ecsKeys).not.toContain('host.name'); + }); + + it('deletes dotted entity field', () => { + const anomaly = buildMockAnomaly(); + const result = transformAnomalyFieldsToEcs(anomaly); + + const ecsKeys = Object.keys(result); + expect(ecsKeys).not.toContain('process.name'); + }); + + it('creates nested influencer fields', () => { + const anomaly = buildMockAnomaly(); + const result = transformAnomalyFieldsToEcs(anomaly); + + expect(result.process.pid).toEqual(['123']); + expect(result.user.name).toEqual(['root']); + expect(result.host.name).toEqual(['rock01']); + }); + + it('creates nested entity field', () => { + const anomaly = buildMockAnomaly(); + const result = transformAnomalyFieldsToEcs(anomaly); + + expect(result.process.name).toEqual(['gzip']); + }); +}); diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/bulk_create_ml_signals.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/bulk_create_ml_signals.ts new file mode 100644 index 0000000000000..1ab34f26d4b70 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/bulk_create_ml_signals.ts @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { flow, set, omit } from 'lodash/fp'; +import { SearchResponse } from 'elasticsearch'; + +import { Logger } from '../../../../../../../../src/core/server'; +import { AlertServices } from '../../../../../../../plugins/alerting/server'; +import { RuleTypeParams } from '../types'; +import { singleBulkCreate } from './single_bulk_create'; +import { AnomalyResults, Anomaly } from '../../machine_learning'; + +interface BulkCreateMlSignalsParams { + someResult: AnomalyResults; + ruleParams: RuleTypeParams; + services: AlertServices; + logger: Logger; + id: string; + signalsIndex: string; + name: string; + createdAt: string; + createdBy: string; + updatedAt: string; + updatedBy: string; + interval: string; + enabled: boolean; + tags: string[]; +} + +interface EcsAnomaly extends Anomaly { + '@timestamp': string; +} + +export const transformAnomalyFieldsToEcs = (anomaly: Anomaly): EcsAnomaly => { + const { + by_field_name: entityName, + by_field_value: entityValue, + influencers, + timestamp, + } = anomaly; + let errantFields = (influencers ?? []).map(influencer => ({ + name: influencer.influencer_field_name, + value: influencer.influencer_field_values, + })); + + if (entityName && entityValue) { + errantFields = [...errantFields, { name: entityName, value: [entityValue] }]; + } + + const omitDottedFields = omit(errantFields.map(field => field.name)); + const setNestedFields = errantFields.map(field => set(field.name, field.value)); + const setTimestamp = set('@timestamp', new Date(timestamp).toISOString()); + + return flow(omitDottedFields, setNestedFields, setTimestamp)(anomaly); +}; + +const transformAnomalyResultsToEcs = (results: AnomalyResults): SearchResponse => { + const transformedHits = results.hits.hits.map(({ _source, ...rest }) => ({ + ...rest, + _source: transformAnomalyFieldsToEcs(_source), + })); + + return { + ...results, + hits: { + ...results.hits, + hits: transformedHits, + }, + }; +}; + +export const bulkCreateMlSignals = async (params: BulkCreateMlSignalsParams) => { + const anomalyResults = params.someResult; + const ecsResults = transformAnomalyResultsToEcs(anomalyResults); + + return singleBulkCreate({ ...params, someResult: ecsResults }); +}; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/find_ml_signals.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/find_ml_signals.ts new file mode 100644 index 0000000000000..b7f752e6ba5e0 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/find_ml_signals.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import dateMath from '@elastic/datemath'; + +import { AlertServices } from '../../../../../../../plugins/alerting/server'; + +import { getAnomalies } from '../../machine_learning'; + +export const findMlSignals = async ( + jobId: string, + anomalyThreshold: number, + from: string, + to: string, + callCluster: AlertServices['callCluster'] +) => { + const params = { + jobIds: [jobId], + threshold: anomalyThreshold, + earliestMs: dateMath.parse(from)?.valueOf() ?? 0, + latestMs: dateMath.parse(to)?.valueOf() ?? 0, + }; + const relevantAnomalies = await getAnomalies(params, callCluster); + + return relevantAnomalies; +}; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/get_filter.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/get_filter.ts index 9c3e15de7ce90..82a50222dc351 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/get_filter.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/get_filter.ts @@ -107,6 +107,11 @@ export const getFilter = async ({ throw new BadRequestError('savedId parameter should be defined'); } } + case 'machine_learning': { + throw new BadRequestError( + 'Unsupported Rule of type "machine_learning" supplied to getFilter' + ); + } } return assertUnreachable(type); }; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.test.ts index bf7a97a29aef3..09daae8485381 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.test.ts @@ -26,8 +26,10 @@ export const mockService = { }; describe('searchAfterAndBulkCreate', () => { + let inputIndexPattern: string[] = []; beforeEach(() => { jest.clearAllMocks(); + inputIndexPattern = ['auditbeat-*']; }); test('if successful with empty search results', async () => { @@ -38,6 +40,7 @@ describe('searchAfterAndBulkCreate', () => { services: mockService, logger: mockLogger, id: sampleRuleGuid, + inputIndexPattern, signalsIndex: DEFAULT_SIGNALS_INDEX, name: 'rule-name', createdAt: '2020-01-28T15:58:34.810Z', @@ -93,6 +96,7 @@ describe('searchAfterAndBulkCreate', () => { services: mockService, logger: mockLogger, id: sampleRuleGuid, + inputIndexPattern, signalsIndex: DEFAULT_SIGNALS_INDEX, name: 'rule-name', createdAt: '2020-01-28T15:58:34.810Z', @@ -119,6 +123,7 @@ describe('searchAfterAndBulkCreate', () => { services: mockService, logger: mockLogger, id: sampleRuleGuid, + inputIndexPattern, signalsIndex: DEFAULT_SIGNALS_INDEX, name: 'rule-name', createdAt: '2020-01-28T15:58:34.810Z', @@ -152,6 +157,7 @@ describe('searchAfterAndBulkCreate', () => { services: mockService, logger: mockLogger, id: sampleRuleGuid, + inputIndexPattern, signalsIndex: DEFAULT_SIGNALS_INDEX, name: 'rule-name', createdAt: '2020-01-28T15:58:34.810Z', @@ -185,6 +191,7 @@ describe('searchAfterAndBulkCreate', () => { services: mockService, logger: mockLogger, id: sampleRuleGuid, + inputIndexPattern, signalsIndex: DEFAULT_SIGNALS_INDEX, name: 'rule-name', createdAt: '2020-01-28T15:58:34.810Z', @@ -220,6 +227,7 @@ describe('searchAfterAndBulkCreate', () => { services: mockService, logger: mockLogger, id: sampleRuleGuid, + inputIndexPattern, signalsIndex: DEFAULT_SIGNALS_INDEX, name: 'rule-name', createdAt: '2020-01-28T15:58:34.810Z', @@ -255,6 +263,7 @@ describe('searchAfterAndBulkCreate', () => { services: mockService, logger: mockLogger, id: sampleRuleGuid, + inputIndexPattern, signalsIndex: DEFAULT_SIGNALS_INDEX, name: 'rule-name', createdAt: '2020-01-28T15:58:34.810Z', @@ -292,6 +301,7 @@ describe('searchAfterAndBulkCreate', () => { services: mockService, logger: mockLogger, id: sampleRuleGuid, + inputIndexPattern, signalsIndex: DEFAULT_SIGNALS_INDEX, name: 'rule-name', createdAt: '2020-01-28T15:58:34.810Z', diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.ts index 1cfd2f812a195..f54ad67af4a48 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.ts @@ -17,6 +17,7 @@ interface SearchAfterAndBulkCreateParams { services: AlertServices; logger: Logger; id: string; + inputIndexPattern: string[]; signalsIndex: string; name: string; createdAt: string; @@ -37,6 +38,7 @@ export const searchAfterAndBulkCreate = async ({ services, logger, id, + inputIndexPattern, signalsIndex, filter, name, @@ -77,7 +79,7 @@ export const searchAfterAndBulkCreate = async ({ // If the total number of hits for the overall search result is greater than // maxSignals, default to requesting a total of maxSignals, otherwise use the // totalHits in the response from the searchAfter query. - const maxTotalHitsSize = totalHits >= ruleParams.maxSignals ? ruleParams.maxSignals : totalHits; + const maxTotalHitsSize = Math.min(totalHits, ruleParams.maxSignals); // number of docs in the current search result let hitsSize = someResult.hits.hits.length; @@ -98,7 +100,9 @@ export const searchAfterAndBulkCreate = async ({ logger.debug(`sortIds: ${sortIds}`); const searchAfterResult: SignalSearchResponse = await singleSearchAfter({ searchAfterSortId: sortId, - ruleParams, + index: inputIndexPattern, + from: ruleParams.from, + to: ruleParams.to, services, logger, filter, diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_params_schema.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_params_schema.ts index adbb5fa618957..7b0546f56dd15 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_params_schema.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_params_schema.ts @@ -14,6 +14,7 @@ import { DEFAULT_MAX_SIGNALS } from '../../../../common/constants'; */ export const signalParamsSchema = () => schema.object({ + anomalyThreshold: schema.maybe(schema.number()), description: schema.string(), note: schema.nullable(schema.string()), falsePositives: schema.arrayOf(schema.string(), { defaultValue: [] }), @@ -27,6 +28,7 @@ export const signalParamsSchema = () => timelineId: schema.nullable(schema.string()), timelineTitle: schema.nullable(schema.string()), meta: schema.nullable(schema.object({}, { unknowns: 'allow' })), + machineLearningJobId: schema.maybe(schema.string()), query: schema.nullable(schema.string()), filters: schema.nullable(schema.arrayOf(schema.object({}, { unknowns: 'allow' }))), maxSignals: schema.number({ defaultValue: DEFAULT_MAX_SIGNALS }), diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts index e3ea121a9ebb1..7a4dcf68e0ca9 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -20,6 +20,8 @@ import { writeGapErrorToSavedObject } from './write_gap_error_to_saved_object'; import { getRuleStatusSavedObjects } from './get_rule_status_saved_objects'; import { getCurrentStatusSavedObject } from './get_current_status_saved_object'; import { writeCurrentStatusSucceeded } from './write_current_status_succeeded'; +import { findMlSignals } from './find_ml_signals'; +import { bulkCreateMlSignals } from './bulk_create_ml_signals'; export const signalRulesAlertType = ({ logger, @@ -38,11 +40,13 @@ export const signalRulesAlertType = ({ }, async executor({ previousStartedAt, alertId, services, params }) { const { + anomalyThreshold, from, ruleId, index, filters, language, + machineLearningJobId, outputIndex, savedId, query, @@ -86,33 +90,70 @@ export const signalRulesAlertType = ({ ruleStatusSavedObjects, name, }); - // set searchAfter page size to be the lesser of default page size or maxSignals. - const searchAfterSize = - DEFAULT_SEARCH_AFTER_PAGE_SIZE <= params.maxSignals - ? DEFAULT_SEARCH_AFTER_PAGE_SIZE - : params.maxSignals; + + const searchAfterSize = Math.min(params.maxSignals, DEFAULT_SEARCH_AFTER_PAGE_SIZE); + let creationSucceeded = false; + try { - const inputIndex = await getInputIndex(services, version, index); - const esFilter = await getFilter({ - type, - filters, - language, - query, - savedId, - services, - index: inputIndex, - }); + if (type === 'machine_learning') { + if (machineLearningJobId == null || anomalyThreshold == null) { + throw new Error( + `Attempted to execute machine learning rule, but it is missing job id and/or anomaly threshold for rule id: "${ruleId}", name: "${name}", signals index: "${outputIndex}", job id: "${machineLearningJobId}", anomaly threshold: "${anomalyThreshold}"` + ); + } - const noReIndex = buildEventsSearchQuery({ - index: inputIndex, - from, - to, - filter: esFilter, - size: searchAfterSize, - searchAfterSortId: undefined, - }); + const anomalyResults = await findMlSignals( + machineLearningJobId, + anomalyThreshold, + from, + to, + services.callCluster + ); + + const anomalyCount = anomalyResults.hits.hits.length; + if (anomalyCount) { + logger.info( + `Found ${anomalyCount} signals from ML anomalies for signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}", pushing signals to index "${outputIndex}"` + ); + } + + creationSucceeded = await bulkCreateMlSignals({ + someResult: anomalyResults, + ruleParams: params, + services, + logger, + id: alertId, + signalsIndex: outputIndex, + name, + createdBy, + createdAt, + updatedBy, + updatedAt, + interval, + enabled, + tags, + }); + } else { + const inputIndex = await getInputIndex(services, version, index); + const esFilter = await getFilter({ + type, + filters, + language, + query, + savedId, + services, + index: inputIndex, + }); + + const noReIndex = buildEventsSearchQuery({ + index: inputIndex, + from, + to, + filter: esFilter, + size: searchAfterSize, + searchAfterSortId: undefined, + }); - try { logger.debug( `Starting signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}"` ); @@ -130,12 +171,13 @@ export const signalRulesAlertType = ({ ); } - const bulkIndexResult = await searchAfterAndBulkCreate({ + creationSucceeded = await searchAfterAndBulkCreate({ someResult: noReIndexResult, ruleParams: params, services, logger, id: alertId, + inputIndexPattern: inputIndex, signalsIndex: outputIndex, filter: esFilter, name, @@ -148,46 +190,35 @@ export const signalRulesAlertType = ({ pageSize: searchAfterSize, tags, }); + } - if (bulkIndexResult) { - logger.debug( - `Finished signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}"` - ); - await writeCurrentStatusSucceeded({ - services, - currentStatusSavedObject, - }); - } else { - await writeSignalRuleExceptionToSavedObject({ - name, - alertId, - currentStatusSavedObject, - logger, - message: `Bulk Indexing signals failed. Check logs for further details \nRule name: "${name}"\nid: "${alertId}"\nrule_id: "${ruleId}"\n`, - services, - ruleStatusSavedObjects, - ruleId: ruleId ?? '(unknown rule id)', - }); - } - } catch (err) { + if (creationSucceeded) { + logger.debug( + `Finished signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}", output_index: "${outputIndex}"` + ); + await writeCurrentStatusSucceeded({ + services, + currentStatusSavedObject, + }); + } else { await writeSignalRuleExceptionToSavedObject({ name, alertId, currentStatusSavedObject, logger, - message: err?.message ?? '(no error message given)', + message: `Bulk Indexing signals failed. Check logs for further details Rule name: "${name}" id: "${alertId}" rule_id: "${ruleId}" output_index: "${outputIndex}"`, services, ruleStatusSavedObjects, ruleId: ruleId ?? '(unknown rule id)', }); } - } catch (exception) { + } catch (error) { await writeSignalRuleExceptionToSavedObject({ name, alertId, currentStatusSavedObject, logger, - message: exception?.message ?? '(no error message given)', + message: error?.message ?? '(no error message given)', services, ruleStatusSavedObjects, ruleId: ruleId ?? '(unknown rule id)', diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/single_search_after.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/single_search_after.test.ts index a5d1f66d3089e..1685c6518def3 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/single_search_after.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/single_search_after.test.ts @@ -6,7 +6,6 @@ import { savedObjectsClientMock } from 'src/core/server/mocks'; import { - sampleRuleAlertParams, sampleDocSearchResultsNoSortId, mockLogger, sampleDocSearchResultsWithSortId, @@ -26,12 +25,13 @@ describe('singleSearchAfter', () => { test('if singleSearchAfter works without a given sort id', async () => { let searchAfterSortId; - const sampleParams = sampleRuleAlertParams(); mockService.callCluster.mockReturnValue(sampleDocSearchResultsNoSortId); await expect( singleSearchAfter({ searchAfterSortId, - ruleParams: sampleParams, + index: [], + from: 'now-360s', + to: 'now', services: mockService, logger: mockLogger, pageSize: 1, @@ -41,11 +41,12 @@ describe('singleSearchAfter', () => { }); test('if singleSearchAfter works with a given sort id', async () => { const searchAfterSortId = '1234567891111'; - const sampleParams = sampleRuleAlertParams(); mockService.callCluster.mockReturnValue(sampleDocSearchResultsWithSortId); const searchAfterResult = await singleSearchAfter({ searchAfterSortId, - ruleParams: sampleParams, + index: [], + from: 'now-360s', + to: 'now', services: mockService, logger: mockLogger, pageSize: 1, @@ -55,14 +56,15 @@ describe('singleSearchAfter', () => { }); test('if singleSearchAfter throws error', async () => { const searchAfterSortId = '1234567891111'; - const sampleParams = sampleRuleAlertParams(); mockService.callCluster.mockImplementation(async () => { throw Error('Fake Error'); }); await expect( singleSearchAfter({ searchAfterSortId, - ruleParams: sampleParams, + index: [], + from: 'now-360s', + to: 'now', services: mockService, logger: mockLogger, pageSize: 1, diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/single_search_after.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/single_search_after.ts index a0e7047ad1cd6..bb12b5a802f8f 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/single_search_after.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/single_search_after.ts @@ -5,14 +5,15 @@ */ import { AlertServices } from '../../../../../../../plugins/alerting/server'; -import { RuleTypeParams } from '../types'; import { Logger } from '../../../../../../../../src/core/server'; import { SignalSearchResponse } from './types'; import { buildEventsSearchQuery } from './build_events_query'; interface SingleSearchAfterParams { searchAfterSortId: string | undefined; - ruleParams: RuleTypeParams; + index: string[]; + from: string; + to: string; services: AlertServices; logger: Logger; pageSize: number; @@ -22,7 +23,9 @@ interface SingleSearchAfterParams { // utilize search_after for paging results into bulk. export const singleSearchAfter = async ({ searchAfterSortId, - ruleParams, + index, + from, + to, services, filter, logger, @@ -33,9 +36,9 @@ export const singleSearchAfter = async ({ } try { const searchAfterQuery = buildEventsSearchQuery({ - index: ruleParams.index, - from: ruleParams.from, - to: ruleParams.to, + index, + from, + to, filter, size: pageSize, searchAfterSortId, diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/types.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/types.ts index eaed3f2ead3a5..1ee3d4f0eb8e4 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/types.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/types.ts @@ -104,7 +104,7 @@ export interface GetResponse { } export type SignalSearchResponse = SearchResponse; -export type SignalSourceHit = SignalSearchResponse['hits']['hits'][0]; +export type SignalSourceHit = SignalSearchResponse['hits']['hits'][number]; export type RuleExecutorOptions = Omit & { params: RuleAlertParams & { diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/types.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/types.ts index fa43ac1debb92..f77924aafadf8 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/types.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/types.ts @@ -22,7 +22,10 @@ export interface ThreatParams { technique: IMitreAttack[]; } +export type RuleType = 'query' | 'saved_query' | 'machine_learning'; + export interface RuleAlertParams { + anomalyThreshold: number | undefined; description: string; note: string | undefined | null; enabled: boolean; @@ -30,11 +33,12 @@ export interface RuleAlertParams { filters: PartialFilter[] | undefined | null; from: string; immutable: boolean; - index: string[]; + index: string[] | undefined | null; interval: string; ruleId: string | undefined | null; language: string | undefined | null; maxSignals: number; + machineLearningJobId: string | undefined; riskScore: number; outputIndex: string; name: string; @@ -48,7 +52,7 @@ export interface RuleAlertParams { timelineId: string | undefined | null; timelineTitle: string | undefined | null; threat: ThreatParams[] | undefined | null; - type: 'query' | 'saved_query'; + type: RuleType; version: number; throttle?: string; } @@ -57,10 +61,12 @@ export type RuleTypeParams = Omit & { + anomaly_threshold: RuleAlertParams['anomalyThreshold']; rule_id: RuleAlertParams['ruleId']; false_positives: RuleAlertParams['falsePositives']; saved_id?: RuleAlertParams['savedId']; timeline_id: RuleAlertParams['timelineId']; timeline_title: RuleAlertParams['timelineTitle']; max_signals: RuleAlertParams['maxSignals']; + machine_learning_job_id: RuleAlertParams['machineLearningJobId']; risk_score: RuleAlertParams['riskScore']; output_index: RuleAlertParams['outputIndex']; created_at: string; diff --git a/x-pack/legacy/plugins/siem/server/lib/machine_learning/index.ts b/x-pack/legacy/plugins/siem/server/lib/machine_learning/index.ts new file mode 100644 index 0000000000000..aa83df15f68d4 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/machine_learning/index.ts @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SearchResponse } from 'elasticsearch'; + +import { AlertServices } from '../../../../../../plugins/alerting/server'; +import { AnomalyRecordDoc as Anomaly } from '../../../../../../plugins/ml/common/types/anomalies'; + +export { Anomaly }; +export type AnomalyResults = SearchResponse; + +export interface AnomaliesSearchParams { + jobIds: string[]; + threshold: number; + earliestMs: number; + latestMs: number; + maxRecords?: number; +} + +export const getAnomalies = async ( + params: AnomaliesSearchParams, + callCluster: AlertServices['callCluster'] +): Promise => { + const boolCriteria = buildCriteria(params); + + return callCluster('search', { + index: '.ml-anomalies-*', + size: params.maxRecords || 100, + body: { + query: { + bool: { + filter: [ + { + query_string: { + query: 'result_type:record', + analyze_wildcard: false, + }, + }, + { + bool: { + must: boolCriteria, + }, + }, + ], + }, + }, + sort: [{ record_score: { order: 'desc' } }], + }, + }); +}; + +const buildCriteria = (params: AnomaliesSearchParams): object[] => { + const { earliestMs, jobIds, latestMs, threshold } = params; + const jobIdsFilterable = jobIds.length > 0 && !(jobIds.length === 1 && jobIds[0] === '*'); + + const boolCriteria: object[] = [ + { + range: { + timestamp: { + gte: earliestMs, + lte: latestMs, + format: 'epoch_millis', + }, + }, + }, + { + range: { + record_score: { + gte: threshold, + }, + }, + }, + ]; + + if (jobIdsFilterable) { + const jobIdFilter = jobIds.map(jobId => `job_id:${jobId}`).join(' OR '); + + boolCriteria.push({ + query_string: { + analyze_wildcard: false, + query: jobIdFilter, + }, + }); + } + + return boolCriteria; +}; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_rules.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_rules.ts index d6a238e5b0940..91088acb7a51c 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_rules.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_rules.ts @@ -18,6 +18,8 @@ import { getSimpleRuleWithoutRuleId, removeServerGeneratedProperties, removeServerGeneratedPropertiesIncludingRuleId, + getSimpleMlRule, + getSimpleMlRuleOutput, } from './utils'; // eslint-disable-next-line import/no-default-export @@ -63,6 +65,20 @@ export default ({ getService }: FtrProviderContext) => { expect(bodyToCompare).to.eql(getSimpleRuleOutput()); }); + it('should create a single rule without an input index', async () => { + const { index, ...payload } = getSimpleRule(); + const { index: _index, ...expected } = getSimpleRuleOutput(); + + const { body } = await supertest + .post(DETECTION_ENGINE_RULES_URL) + .set('kbn-xsrf', 'true') + .send(payload) + .expect(200); + + const bodyToCompare = removeServerGeneratedProperties(body); + expect(bodyToCompare).to.eql(expected); + }); + it('should create a single rule without a rule_id', async () => { const { body } = await supertest .post(DETECTION_ENGINE_RULES_URL) @@ -74,6 +90,17 @@ export default ({ getService }: FtrProviderContext) => { expect(bodyToCompare).to.eql(getSimpleRuleOutputWithoutRuleId()); }); + it('should create a single Machine Learning rule', async () => { + const { body } = await supertest + .post(DETECTION_ENGINE_RULES_URL) + .set('kbn-xsrf', 'true') + .send(getSimpleMlRule()) + .expect(200); + + const bodyToCompare = removeServerGeneratedProperties(body); + expect(bodyToCompare).to.eql(getSimpleMlRuleOutput()); + }); + it('should cause a 409 conflict if we attempt to create the same rule_id twice', async () => { await supertest .post(DETECTION_ENGINE_RULES_URL) diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/utils.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/utils.ts index 1570124cdb92b..8847a2fdb21af 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/utils.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/utils.ts @@ -49,10 +49,26 @@ export const getSimpleRule = (ruleId = 'rule-1'): Partial = risk_score: 1, rule_id: ruleId, severity: 'high', + index: ['auditbeat-*'], type: 'query', query: 'user.name: root or user.name: admin', }); +/** + * This is a representative ML rule payload as expected by the server + * @param ruleId + */ +export const getSimpleMlRule = (ruleId = 'rule-1'): Partial => ({ + name: 'Simple ML Rule', + description: 'Simple Machine Learning Rule', + anomaly_threshold: 44, + risk_score: 1, + rule_id: ruleId, + severity: 'high', + machine_learning_job_id: 'some_job_id', + type: 'machine_learning', +}); + export const getSignalStatus = () => ({ aggs: { statuses: { terms: { field: 'signal.status', size: 10 } } }, }); @@ -118,6 +134,7 @@ export const getSimpleRuleOutput = (ruleId = 'rule-1'): Partial => { + const rule = getSimpleRuleOutput(ruleId); + const { query, language, index, ...rest } = rule; + + return { + ...rest, + name: 'Simple ML Rule', + description: 'Simple Machine Learning Rule', + anomaly_threshold: 44, + machine_learning_job_id: 'some_job_id', + type: 'machine_learning', + }; +}; + /** * Remove all alerts from the .kibana index * @param es The ElasticSearch handle