diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts index bfe450d240b08..a5398f291627c 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts @@ -465,26 +465,56 @@ export type Threats = t.TypeOf; export const threatsOrUndefined = t.union([threats, t.undefined]); export type ThreatsOrUndefined = t.TypeOf; +export const thresholdField = t.exact( + t.type({ + field: t.union([t.string, t.array(t.string)]), // Covers pre- and post-7.12 + value: PositiveIntegerGreaterThanZero, + }) +); +export type ThresholdField = t.TypeOf; + +export const thresholdFieldNormalized = t.exact( + t.type({ + field: t.array(t.string), + value: PositiveIntegerGreaterThanZero, + }) +); +export type ThresholdFieldNormalized = t.TypeOf; + +export const thresholdCardinalityField = t.exact( + t.type({ + field: t.string, + value: PositiveInteger, + }) +); +export type ThresholdCardinalityField = t.TypeOf; + export const threshold = t.intersection([ - t.exact( - t.type({ - field: t.union([t.string, t.array(t.string)]), - value: PositiveIntegerGreaterThanZero, - }) - ), + thresholdField, t.exact( t.partial({ - cardinality_field: t.union([t.string, t.array(t.string), t.undefined, t.null]), - cardinality_value: t.union([PositiveInteger, t.undefined, t.null]), // TODO: cardinality_value should be set if cardinality_field is set + cardinality: t.union([t.array(thresholdCardinalityField), t.null]), }) ), ]); -// TODO: codec to transform threshold field string to string[] ? export type Threshold = t.TypeOf; export const thresholdOrUndefined = t.union([threshold, t.undefined]); export type ThresholdOrUndefined = t.TypeOf; +export const thresholdNormalized = t.intersection([ + thresholdFieldNormalized, + t.exact( + t.partial({ + cardinality: t.union([t.array(thresholdCardinalityField), t.null]), + }) + ), +]); +export type ThresholdNormalized = t.TypeOf; + +export const thresholdNormalizedOrUndefined = t.union([thresholdNormalized, t.undefined]); +export type ThresholdNormalizedOrUndefined = t.TypeOf; + export const created_at = IsoDateString; export const updated_at = IsoDateString; export const updated_by = t.string; diff --git a/x-pack/plugins/security_solution/common/detection_engine/utils.test.ts b/x-pack/plugins/security_solution/common/detection_engine/utils.test.ts index 72c3a49bb66b8..9377255dc85d5 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/utils.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/utils.test.ts @@ -5,7 +5,13 @@ * 2.0. */ -import { hasEqlSequenceQuery, hasLargeValueList, hasNestedEntry, isThreatMatchRule } from './utils'; +import { + hasEqlSequenceQuery, + hasLargeValueList, + hasNestedEntry, + isThreatMatchRule, + normalizeThresholdField, +} from './utils'; import { EntriesArray } from '../shared_imports'; describe('#hasLargeValueList', () => { @@ -151,3 +157,21 @@ describe('#hasEqlSequenceQuery', () => { }); }); }); + +describe('normalizeThresholdField', () => { + it('converts a string to a string array', () => { + expect(normalizeThresholdField('host.name')).toEqual(['host.name']); + }); + it('returns a string array when a string array is passed in', () => { + expect(normalizeThresholdField(['host.name'])).toEqual(['host.name']); + }); + it('converts undefined to an empty array', () => { + expect(normalizeThresholdField(undefined)).toEqual([]); + }); + it('converts null to an empty array', () => { + expect(normalizeThresholdField(null)).toEqual([]); + }); + it('converts an empty string to an empty array', () => { + expect(normalizeThresholdField('')).toEqual([]); + }); +}); diff --git a/x-pack/plugins/security_solution/common/detection_engine/utils.ts b/x-pack/plugins/security_solution/common/detection_engine/utils.ts index 79b912e082fdb..838bac542bb87 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/utils.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/utils.ts @@ -5,6 +5,8 @@ * 2.0. */ +import { isEmpty } from 'lodash'; + import { CreateExceptionListItemSchema, EntriesArray, @@ -42,3 +44,13 @@ export const isQueryRule = (ruleType: Type | undefined): boolean => ruleType === 'query' || ruleType === 'saved_query'; export const isThreatMatchRule = (ruleType: Type | undefined): boolean => ruleType === 'threat_match'; + +export const normalizeThresholdField = ( + thresholdField: string | string[] | null | undefined +): string[] => { + return Array.isArray(thresholdField) + ? thresholdField + : isEmpty(thresholdField) + ? [] + : [thresholdField!]; +}; diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/matrix_histogram/index.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/matrix_histogram/index.ts index c71108d58d980..0b59170dfc778 100644 --- a/x-pack/plugins/security_solution/common/search_strategy/security_solution/matrix_histogram/index.ts +++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/matrix_histogram/index.ts @@ -40,8 +40,10 @@ export interface MatrixHistogramRequestOptions extends RequestBasicOptions { | { field: string | string[] | undefined; value: number; - cardinality_field?: string | undefined; - cardinality_value?: number | undefined; + cardinality?: { + field: string[]; + value: number; + }; } | undefined; inspect?: Maybe; diff --git a/x-pack/plugins/security_solution/cypress/integration/detection_rules/threshold_rule.spec.ts b/x-pack/plugins/security_solution/cypress/integration/detection_rules/threshold_rule.spec.ts index 572422a4936df..56f316087ae8a 100644 --- a/x-pack/plugins/security_solution/cypress/integration/detection_rules/threshold_rule.spec.ts +++ b/x-pack/plugins/security_solution/cypress/integration/detection_rules/threshold_rule.spec.ts @@ -79,103 +79,100 @@ import { loginAndWaitForPageWithoutDateRange } from '../../tasks/login'; import { DETECTIONS_URL } from '../../urls/navigation'; -// Skipped until post-FF for 7.12 -describe.skip('Threshold Rules', () => { - describe('Detection rules, threshold', () => { - const expectedUrls = newThresholdRule.referenceUrls.join(''); - const expectedFalsePositives = newThresholdRule.falsePositivesExamples.join(''); - const expectedTags = newThresholdRule.tags.join(''); - const expectedMitre = formatMitreAttackDescription(newThresholdRule.mitre); - - const rule = { ...newThresholdRule }; - - beforeEach(() => { - cleanKibana(); - createTimeline(newThresholdRule.timeline).then((response) => { - rule.timeline.id = response.body.data.persistTimeline.timeline.savedObjectId; - }); +describe('Detection rules, threshold', () => { + const expectedUrls = newThresholdRule.referenceUrls.join(''); + const expectedFalsePositives = newThresholdRule.falsePositivesExamples.join(''); + const expectedTags = newThresholdRule.tags.join(''); + const expectedMitre = formatMitreAttackDescription(newThresholdRule.mitre); + + const rule = { ...newThresholdRule }; + + beforeEach(() => { + cleanKibana(); + createTimeline(newThresholdRule.timeline).then((response) => { + rule.timeline.id = response.body.data.persistTimeline.timeline.savedObjectId; }); + }); - it('Creates and activates a new threshold rule', () => { - loginAndWaitForPageWithoutDateRange(DETECTIONS_URL); - waitForAlertsPanelToBeLoaded(); - waitForAlertsIndexToBeCreated(); - goToManageAlertsDetectionRules(); - waitForRulesTableToBeLoaded(); - goToCreateNewRule(); - selectThresholdRuleType(); - fillDefineThresholdRuleAndContinue(rule); - fillAboutRuleAndContinue(rule); - fillScheduleRuleAndContinue(rule); - createAndActivateRule(); - - cy.get(CUSTOM_RULES_BTN).should('have.text', 'Custom rules (1)'); - - changeRowsPerPageTo300(); - - const expectedNumberOfRules = 1; - cy.get(RULES_TABLE).then(($table) => { - cy.wrap($table.find(RULES_ROW).length).should('eql', expectedNumberOfRules); - }); + it('Creates and activates a new threshold rule', () => { + loginAndWaitForPageWithoutDateRange(DETECTIONS_URL); + waitForAlertsPanelToBeLoaded(); + waitForAlertsIndexToBeCreated(); + goToManageAlertsDetectionRules(); + waitForRulesTableToBeLoaded(); + goToCreateNewRule(); + selectThresholdRuleType(); + fillDefineThresholdRuleAndContinue(rule); + fillAboutRuleAndContinue(rule); + fillScheduleRuleAndContinue(rule); + createAndActivateRule(); + + cy.get(CUSTOM_RULES_BTN).should('have.text', 'Custom rules (1)'); + + changeRowsPerPageTo300(); + + const expectedNumberOfRules = 1; + cy.get(RULES_TABLE).then(($table) => { + cy.wrap($table.find(RULES_ROW).length).should('eql', expectedNumberOfRules); + }); - filterByCustomRules(); + filterByCustomRules(); - cy.get(RULES_TABLE).then(($table) => { - cy.wrap($table.find(RULES_ROW).length).should('eql', 1); - }); - cy.get(RULE_NAME).should('have.text', rule.name); - cy.get(RISK_SCORE).should('have.text', rule.riskScore); - cy.get(SEVERITY).should('have.text', rule.severity); - cy.get(RULE_SWITCH).should('have.attr', 'aria-checked', 'true'); - - goToRuleDetails(); - - cy.get(RULE_NAME_HEADER).should('have.text', `${rule.name}`); - cy.get(ABOUT_RULE_DESCRIPTION).should('have.text', rule.description); - cy.get(ABOUT_DETAILS).within(() => { - getDetails(SEVERITY_DETAILS).should('have.text', rule.severity); - getDetails(RISK_SCORE_DETAILS).should('have.text', rule.riskScore); - getDetails(REFERENCE_URLS_DETAILS).should((details) => { - expect(removeExternalLinkText(details.text())).equal(expectedUrls); - }); - getDetails(FALSE_POSITIVES_DETAILS).should('have.text', expectedFalsePositives); - getDetails(MITRE_ATTACK_DETAILS).should((mitre) => { - expect(removeExternalLinkText(mitre.text())).equal(expectedMitre); - }); - getDetails(TAGS_DETAILS).should('have.text', expectedTags); - }); - cy.get(INVESTIGATION_NOTES_TOGGLE).click({ force: true }); - cy.get(ABOUT_INVESTIGATION_NOTES).should('have.text', INVESTIGATION_NOTES_MARKDOWN); - cy.get(DEFINITION_DETAILS).within(() => { - getDetails(INDEX_PATTERNS_DETAILS).should('have.text', indexPatterns.join('')); - getDetails(CUSTOM_QUERY_DETAILS).should('have.text', rule.customQuery); - getDetails(RULE_TYPE_DETAILS).should('have.text', 'Threshold'); - getDetails(TIMELINE_TEMPLATE_DETAILS).should('have.text', 'None'); - getDetails(THRESHOLD_DETAILS).should( - 'have.text', - `Results aggregated by ${rule.thresholdField} >= ${rule.threshold}` - ); + cy.get(RULES_TABLE).then(($table) => { + cy.wrap($table.find(RULES_ROW).length).should('eql', 1); + }); + cy.get(RULE_NAME).should('have.text', rule.name); + cy.get(RISK_SCORE).should('have.text', rule.riskScore); + cy.get(SEVERITY).should('have.text', rule.severity); + cy.get(RULE_SWITCH).should('have.attr', 'aria-checked', 'true'); + + goToRuleDetails(); + + cy.get(RULE_NAME_HEADER).should('have.text', `${rule.name}`); + cy.get(ABOUT_RULE_DESCRIPTION).should('have.text', rule.description); + cy.get(ABOUT_DETAILS).within(() => { + getDetails(SEVERITY_DETAILS).should('have.text', rule.severity); + getDetails(RISK_SCORE_DETAILS).should('have.text', rule.riskScore); + getDetails(REFERENCE_URLS_DETAILS).should((details) => { + expect(removeExternalLinkText(details.text())).equal(expectedUrls); }); - cy.get(SCHEDULE_DETAILS).within(() => { - getDetails(RUNS_EVERY_DETAILS).should( - 'have.text', - `${rule.runsEvery.interval}${rule.runsEvery.type}` - ); - getDetails(ADDITIONAL_LOOK_BACK_DETAILS).should( - 'have.text', - `${rule.lookBack.interval}${rule.lookBack.type}` - ); + getDetails(FALSE_POSITIVES_DETAILS).should('have.text', expectedFalsePositives); + getDetails(MITRE_ATTACK_DETAILS).should((mitre) => { + expect(removeExternalLinkText(mitre.text())).equal(expectedMitre); }); + getDetails(TAGS_DETAILS).should('have.text', expectedTags); + }); + cy.get(INVESTIGATION_NOTES_TOGGLE).click({ force: true }); + cy.get(ABOUT_INVESTIGATION_NOTES).should('have.text', INVESTIGATION_NOTES_MARKDOWN); + cy.get(DEFINITION_DETAILS).within(() => { + getDetails(INDEX_PATTERNS_DETAILS).should('have.text', indexPatterns.join('')); + getDetails(CUSTOM_QUERY_DETAILS).should('have.text', rule.customQuery); + getDetails(RULE_TYPE_DETAILS).should('have.text', 'Threshold'); + getDetails(TIMELINE_TEMPLATE_DETAILS).should('have.text', 'None'); + getDetails(THRESHOLD_DETAILS).should( + 'have.text', + `Results aggregated by ${rule.thresholdField} >= ${rule.threshold}` + ); + }); + cy.get(SCHEDULE_DETAILS).within(() => { + getDetails(RUNS_EVERY_DETAILS).should( + 'have.text', + `${rule.runsEvery.interval}${rule.runsEvery.type}` + ); + getDetails(ADDITIONAL_LOOK_BACK_DETAILS).should( + 'have.text', + `${rule.lookBack.interval}${rule.lookBack.type}` + ); + }); - waitForTheRuleToBeExecuted(); - waitForAlertsToPopulate(); + waitForTheRuleToBeExecuted(); + waitForAlertsToPopulate(); - cy.get(NUMBER_OF_ALERTS).should(($count) => expect(+$count.text()).to.be.lt(100)); - cy.get(ALERT_RULE_NAME).first().should('have.text', rule.name); - cy.get(ALERT_RULE_VERSION).first().should('have.text', '1'); - cy.get(ALERT_RULE_METHOD).first().should('have.text', 'threshold'); - cy.get(ALERT_RULE_SEVERITY).first().should('have.text', rule.severity.toLowerCase()); - cy.get(ALERT_RULE_RISK_SCORE).first().should('have.text', rule.riskScore); - }); + cy.get(NUMBER_OF_ALERTS).should(($count) => expect(+$count.text()).to.be.lt(100)); + cy.get(ALERT_RULE_NAME).first().should('have.text', rule.name); + cy.get(ALERT_RULE_VERSION).first().should('have.text', '1'); + cy.get(ALERT_RULE_METHOD).first().should('have.text', 'threshold'); + cy.get(ALERT_RULE_SEVERITY).first().should('have.text', rule.severity.toLowerCase()); + cy.get(ALERT_RULE_RISK_SCORE).first().should('have.text', rule.riskScore); }); }); diff --git a/x-pack/plugins/security_solution/public/common/components/matrix_histogram/types.ts b/x-pack/plugins/security_solution/public/common/components/matrix_histogram/types.ts index b993bcda56b8e..d846d887cb681 100644 --- a/x-pack/plugins/security_solution/public/common/components/matrix_histogram/types.ts +++ b/x-pack/plugins/security_solution/public/common/components/matrix_histogram/types.ts @@ -78,8 +78,10 @@ export interface MatrixHistogramQueryProps { | { field: string | string[] | undefined; value: number; - cardinality_field?: string | undefined; - cardinality_value?: number | undefined; + cardinality?: { + field: string[]; + value: number; + }; } | undefined; skip?: boolean; diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/index.test.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/index.test.tsx index bb87242d9bf10..03692457eca5e 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/index.test.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/index.test.tsx @@ -291,8 +291,10 @@ describe('PreviewQuery', () => { threshold={{ field: 'agent.hostname', value: 200, - cardinality_field: 'user.name', - cardinality_value: 2, + cardinality: { + field: ['user.name'], + value: 2, + }, }} isDisabled={false} /> @@ -338,8 +340,10 @@ describe('PreviewQuery', () => { threshold={{ field: 'agent.hostname', value: 200, - cardinality_field: 'user.name', - cardinality_value: 2, + cardinality: { + field: ['user.name'], + value: 2, + }, }} isDisabled={false} /> @@ -382,8 +386,10 @@ describe('PreviewQuery', () => { threshold={{ field: undefined, value: 200, - cardinality_field: 'user.name', - cardinality_value: 2, + cardinality: { + field: ['user.name'], + value: 2, + }, }} isDisabled={false} /> @@ -414,8 +420,10 @@ describe('PreviewQuery', () => { threshold={{ field: ' ', value: 200, - cardinality_field: 'user.name', - cardinality_value: 2, + cardinality: { + field: ['user.name'], + value: 2, + }, }} isDisabled={false} /> diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/index.tsx index 377259fc9b212..e3920856ea19e 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/index.tsx @@ -60,8 +60,10 @@ export type Threshold = | { field: string | string[] | undefined; value: number; - cardinality_field: string | undefined; - cardinality_value: number | undefined; + cardinality?: { + field: string[]; + value: number; + }; } | undefined; diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/reducer.test.ts b/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/reducer.test.ts index d1a9e5c5f768f..b0728cd8cc827 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/reducer.test.ts +++ b/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/reducer.test.ts @@ -337,8 +337,10 @@ describe('queryPreviewReducer', () => { threshold: { field: 'agent.hostname', value: 200, - cardinality_field: 'user.name', - cardinality_value: 2, + cardinality: { + field: ['user.name'], + value: 2, + }, }, ruleType: 'threshold', }); @@ -355,8 +357,10 @@ describe('queryPreviewReducer', () => { threshold: { field: undefined, value: 200, - cardinality_field: 'user.name', - cardinality_value: 2, + cardinality: { + field: ['user.name'], + value: 2, + }, }, ruleType: 'threshold', }); @@ -373,8 +377,10 @@ describe('queryPreviewReducer', () => { threshold: { field: ' ', value: 200, - cardinality_field: 'user.name', - cardinality_value: 2, + cardinality: { + field: ['user.name'], + value: 2, + }, }, ruleType: 'threshold', }); @@ -391,8 +397,10 @@ describe('queryPreviewReducer', () => { threshold: { field: 'agent.hostname', value: 200, - cardinality_field: 'user.name', - cardinality_value: 2, + cardinality: { + field: ['user.name'], + value: 2, + }, }, ruleType: 'eql', }); @@ -408,8 +416,10 @@ describe('queryPreviewReducer', () => { threshold: { field: 'agent.hostname', value: 200, - cardinality_field: 'user.name', - cardinality_value: 2, + cardinality: { + field: ['user.name'], + value: 2, + }, }, ruleType: 'query', }); @@ -425,8 +435,10 @@ describe('queryPreviewReducer', () => { threshold: { field: 'agent.hostname', value: 200, - cardinality_field: 'user.name', - cardinality_value: 2, + cardinality: { + field: ['user.name'], + value: 2, + }, }, ruleType: 'saved_query', }); diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx index 4c7a34dbdf080..c6bb35fdb7266 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx @@ -82,8 +82,10 @@ const stepDefineDefaultValue: DefineStepRule = { threshold: { field: [], value: '200', - cardinality_field: [], - cardinality_value: '2', + cardinality: { + field: [], + value: '', + }, }, timeline: { id: null, @@ -154,15 +156,15 @@ const StepDefineRuleComponent: FC = ({ threatIndex: formThreatIndex, 'threshold.field': formThresholdField, 'threshold.value': formThresholdValue, - 'threshold.cardinality_field': formThresholdCardinalityField, - 'threshold.cardinality_value': formThresholdCardinalityValue, + 'threshold.cardinality.field': formThresholdCardinalityField, + 'threshold.cardinality.value': formThresholdCardinalityValue, }, ] = useFormData< DefineStepRule & { 'threshold.field': string[] | undefined; 'threshold.value': number | undefined; - 'threshold.cardinality_field': string[] | undefined; - 'threshold.cardinality_value': number | undefined; + 'threshold.cardinality.field': string[] | undefined; + 'threshold.cardinality.value': number | undefined; } >({ form, @@ -172,8 +174,8 @@ const StepDefineRuleComponent: FC = ({ 'queryBar', 'threshold.field', 'threshold.value', - 'threshold.cardinality_field', - 'threshold.cardinality_value', + 'threshold.cardinality.field', + 'threshold.cardinality.value', 'threatIndex', ], }); @@ -289,15 +291,14 @@ const StepDefineRuleComponent: FC = ({ }, []); const thresholdFormValue = useMemo((): Threshold | undefined => { - return formThresholdValue != null && - formThresholdField != null && - formThresholdCardinalityField != null && - formThresholdCardinalityValue != null + return formThresholdValue != null ? { - field: formThresholdField[0], + field: formThresholdField ?? [], value: formThresholdValue, - cardinality_field: formThresholdCardinalityField[0], - cardinality_value: formThresholdCardinalityValue, + cardinality: { + field: formThresholdCardinalityField ?? [], + value: formThresholdCardinalityValue ?? 0, // FIXME + }, } : undefined; }, [ @@ -460,10 +461,10 @@ const StepDefineRuleComponent: FC = ({ path: 'threshold.value', }, thresholdCardinalityField: { - path: 'threshold.cardinality_field', + path: 'threshold.cardinality.field', }, thresholdCardinalityValue: { - path: 'threshold.cardinality_value', + path: 'threshold.cardinality.value', }, }} > diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/schema.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/schema.tsx index a5352ede83d51..194584ec8eb87 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/schema.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/schema.tsx @@ -239,52 +239,85 @@ export const schema: FormSchema = { }, ], }, - cardinality_field: { - type: FIELD_TYPES.COMBO_BOX, - label: i18n.translate( - 'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldThresholdCardinalityFieldLabel', - { - defaultMessage: 'Count', - } - ), - helpText: i18n.translate( - 'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldThresholdFieldCardinalityFieldHelpText', - { - defaultMessage: 'Select a field to check cardinality', - } - ), - }, - cardinality_value: { - type: FIELD_TYPES.NUMBER, - label: i18n.translate( - 'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldThresholdCardinalityValueFieldLabel', - { - defaultMessage: 'Unique values', - } - ), - validations: [ - { - validator: ( - ...args: Parameters - ): ReturnType> | undefined => { - const [{ formData }] = args; - const needsValidation = isThresholdRule(formData.ruleType); - if (!needsValidation) { - return; - } - return fieldValidators.numberGreaterThanField({ - than: 1, - message: i18n.translate( - 'xpack.securitySolution.detectionEngine.validations.thresholdValueFieldData.numberGreaterThanOrEqualOneErrorMessage', - { - defaultMessage: 'Value must be greater than or equal to one.', - } - ), - allowEquality: true, - })(...args); + cardinality: { + field: { + defaultValue: [], + fieldsToValidateOnChange: ['threshold.cardinality.field', 'threshold.cardinality.value'], + type: FIELD_TYPES.COMBO_BOX, + label: i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldThresholdCardinalityFieldLabel', + { + defaultMessage: 'Count', + } + ), + validations: [ + { + validator: ( + ...args: Parameters + ): ReturnType> | undefined => { + const [{ formData }] = args; + const needsValidation = isThresholdRule(formData.ruleType); + if (!needsValidation) { + return; + } + if ( + isEmpty(formData['threshold.cardinality.field']) && + !isEmpty(formData['threshold.cardinality.value']) + ) { + return fieldValidators.emptyField( + i18n.translate( + 'xpack.securitySolution.detectionEngine.validations.thresholdCardinalityFieldFieldData.thresholdCardinalityFieldNotSuppliedMessage', + { + defaultMessage: 'A Cardinality Field is required.', + } + ) + )(...args); + } + }, }, - }, - ], + ], + helpText: i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldThresholdFieldCardinalityFieldHelpText', + { + defaultMessage: 'Select a field to check cardinality', + } + ), + }, + value: { + fieldsToValidateOnChange: ['threshold.cardinality.field', 'threshold.cardinality.value'], + type: FIELD_TYPES.NUMBER, + label: i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldThresholdCardinalityValueFieldLabel', + { + defaultMessage: 'Unique values', + } + ), + validations: [ + { + validator: ( + ...args: Parameters + ): ReturnType> | undefined => { + const [{ formData }] = args; + const needsValidation = isThresholdRule(formData.ruleType); + if (!needsValidation) { + return; + } + if (!isEmpty(formData['threshold.cardinality.field'])) { + return fieldValidators.numberGreaterThanField({ + than: 1, + message: i18n.translate( + 'xpack.securitySolution.detectionEngine.validations.thresholdCardinalityValueFieldData.numberGreaterThanOrEqualOneErrorMessage', + { + defaultMessage: 'Value must be greater than or equal to one.', + } + ), + allowEquality: true, + })(...args); + } + }, + }, + ], + }, }, }, threatIndex: { diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/threshold_input/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/threshold_input/index.tsx index 287c99dce3e60..77c88918abf9c 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/threshold_input/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/threshold_input/index.tsx @@ -19,8 +19,10 @@ const FIELD_COMBO_BOX_WIDTH = 410; export interface FieldValueThreshold { field: string[]; value: string; - cardinality_field: string[]; - cardinality_value: string; + cardinality?: { + field: string[]; + value: string; + }; } interface ThresholdInputProps { diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/__mocks__/mock.ts b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/__mocks__/mock.ts index 8bdbb1a74c73a..ee2c2c48d22ee 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/__mocks__/mock.ts +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/__mocks__/mock.ts @@ -143,8 +143,12 @@ export const mockRuleWithEverything = (id: string): Rule => ({ threshold: { field: ['host.name'], value: 50, - cardinality_field: ['process.name'], - cardinality_value: 2, + cardinality: [ + { + field: 'process.name', + value: 2, + }, + ], }, throttle: 'no_actions', timestamp_override: 'event.ingested', @@ -192,10 +196,12 @@ export const mockDefineStepRule = (): DefineStepRule => ({ }, threatIndex: [], threshold: { - field: [''], + field: [], value: '100', - cardinality_field: [''], - cardinality_value: '2', + cardinality: { + field: ['process.name'], + value: '2', + }, }, }); diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts index b8824d2b8798e..64dfac5787f23 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts @@ -221,8 +221,16 @@ export const formatDefineStepData = (defineStepData: DefineStepRule): DefineStep threshold: { field: ruleFields.threshold?.field ?? [], value: parseInt(ruleFields.threshold?.value, 10) ?? 0, - cardinality_field: ruleFields.threshold.cardinality_field[0] ?? '', - cardinality_value: parseInt(ruleFields.threshold?.cardinality_value, 10) ?? 0, + cardinality: + !isEmpty(ruleFields.threshold.cardinality?.field) && + ruleFields.threshold.cardinality?.value != null + ? [ + { + field: ruleFields.threshold.cardinality.field[0], + value: parseInt(ruleFields.threshold.cardinality.value, 10), + }, + ] + : [], }, }), } diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx index 29d1512030e74..9c2e7751753ee 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx @@ -84,8 +84,10 @@ describe('rule helpers', () => { threshold: { field: ['host.name'], value: '50', - cardinality_field: ['process.name'], - cardinality_value: '2', + cardinality: { + field: ['process.name'], + value: '2', + }, }, threatIndex: [], threatMapping: [], @@ -215,8 +217,6 @@ describe('rule helpers', () => { threshold: { field: [], value: '100', - cardinality_field: [], - cardinality_value: '0', }, threatIndex: [], threatMapping: [], @@ -259,8 +259,6 @@ describe('rule helpers', () => { threshold: { field: [], value: '100', - cardinality_field: [], - cardinality_value: '0', }, threatIndex: [], threatMapping: [], diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx index 7c3930bb21d9a..9bc3ab9103b42 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx @@ -13,6 +13,7 @@ import { useLocation } from 'react-router-dom'; import styled from 'styled-components'; import { EuiFlexItem } from '@elastic/eui'; import { ActionVariables } from '../../../../../../triggers_actions_ui/public'; +import { normalizeThresholdField } from '../../../../../common/detection_engine/utils'; import { RuleAlertAction } from '../../../../../common/detection_engine/types'; import { assertUnreachable } from '../../../../../common/utility_types'; import { transformRuleToAlertAction } from '../../../../../common/detection_engine/transform_actions'; @@ -99,18 +100,16 @@ export const getDefineStepsData = (rule: Rule): DefineStepRule => ({ title: rule.timeline_title ?? null, }, threshold: { - field: rule.threshold?.field - ? Array.isArray(rule.threshold.field) - ? rule.threshold.field - : [rule.threshold.field] - : [], + field: normalizeThresholdField(rule.threshold?.field), value: `${rule.threshold?.value || 100}`, - cardinality_field: Array.isArray(rule.threshold?.cardinality_field) - ? rule.threshold!.cardinality_field - : rule.threshold?.cardinality_field != null - ? [rule.threshold!.cardinality_field] - : [], - cardinality_value: `${rule.threshold?.cardinality_value ?? 0}`, + ...(rule.threshold?.cardinality?.length + ? { + cardinality: { + field: [`${rule.threshold.cardinality[0].field}`], + value: `${rule.threshold.cardinality[0].value}`, + }, + } + : {}), }, }); diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts index 668ca556539ad..b6ea18d0494e5 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts @@ -160,8 +160,10 @@ export interface DefineStepRuleJson { threshold?: { field: string[]; value: number; - cardinality_field: string; - cardinality_value: number; + cardinality: Array<{ + field: string; + value: number; + }>; }; threat_query?: string; threat_mapping?: ThreatMapping; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_converters.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_converters.ts index e9a75af14310e..58ce1e7e14460 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_converters.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_converters.ts @@ -7,6 +7,7 @@ import uuid from 'uuid'; import { InternalRuleCreate, InternalRuleResponse, TypeSpecificRuleParams } from './rule_schemas'; +import { normalizeThresholdField } from '../../../../common/detection_engine/utils'; import { assertUnreachable } from '../../../../common/utility_types'; import { CreateRulesSchema, @@ -207,7 +208,10 @@ export const typeSpecificCamelToSnake = (params: TypeSpecificRuleParams): Respon query: params.query, filters: params.filters, saved_id: params.savedId, - threshold: params.threshold, + threshold: { + ...params.threshold, + field: normalizeThresholdField(params.threshold.field), + }, }; } case 'machine_learning': { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts index 977b38e59f856..2d6e90443c00f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts @@ -434,8 +434,12 @@ export const sampleThresholdSignalHit = (): SignalHit => ({ threshold: { field: ['host.name'], value: 5, - cardinality_field: 'process.name', - cardinality_value: 2, + cardinality: [ + { + field: 'process.name', + value: 2, + }, + ], }, updated_by: 'elastic_kibana', tags: ['some fake tag 1', 'some fake tag 2'], @@ -460,6 +464,25 @@ export const sampleThresholdSignalHit = (): SignalHit => ({ }, }); +const sampleThresholdHit = sampleThresholdSignalHit(); +export const sampleLegacyThresholdSignalHit = (): unknown => ({ + ...sampleThresholdHit, + signal: { + ...sampleThresholdHit.signal, + rule: { + ...sampleThresholdHit.signal.rule, + threshold: { + field: 'host.name', + value: 5, + }, + }, + threshold_result: { + count: 72, + value: 'a hostname', + }, + }, +}); + export const sampleWrappedThresholdSignalHit = (): WrappedSignalHit => { return { _index: 'myFakeSignalIndex', @@ -468,6 +491,14 @@ export const sampleWrappedThresholdSignalHit = (): WrappedSignalHit => { }; }; +export const sampleWrappedLegacyThresholdSignalHit = (): WrappedSignalHit => { + return { + _index: 'myFakeSignalIndex', + _id: 'adb9d636-fbbe-4962-ac1c-e282f3ec5879', + _source: sampleLegacyThresholdSignalHit() as SignalHit, + }; +}; + export const sampleBulkCreateDuplicateResult = { took: 60, errors: true, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.test.ts index 56d71048bb81b..130077d2fdf2b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.test.ts @@ -9,15 +9,172 @@ import { loggingSystemMock } from '../../../../../../../src/core/server/mocks'; import { sampleDocNoSortId, sampleDocSearchResultsNoSortId } from './__mocks__/es_results'; import { transformThresholdResultsToEcs } from './bulk_create_threshold_signals'; import { calculateThresholdSignalUuid } from './utils'; -import { Threshold } from '../../../../common/detection_engine/schemas/common/schemas'; +import { normalizeThresholdField } from '../../../../common/detection_engine/utils'; +import { + Threshold, + ThresholdNormalized, +} from '../../../../common/detection_engine/schemas/common/schemas'; -describe('transformThresholdResultsToEcs', () => { - it('should return transformed threshold results', () => { +describe('transformThresholdNormalizedResultsToEcs', () => { + it('should return transformed threshold results for pre-7.12 rules', () => { const threshold: Threshold = { + field: 'source.ip', + value: 1, + }; + const startedAt = new Date('2020-12-17T16:27:00Z'); + const transformedResults = transformThresholdResultsToEcs( + { + ...sampleDocSearchResultsNoSortId('abcd'), + aggregations: { + 'threshold_0:source.ip': { + buckets: [ + { + key: '127.0.0.1', + doc_count: 15, + top_threshold_hits: { + hits: { + hits: [sampleDocNoSortId('abcd')], + }, + }, + }, + ], + }, + }, + }, + 'test', + startedAt, + undefined, + loggingSystemMock.createLogger(), + { + ...threshold, + field: normalizeThresholdField(threshold.field), + }, + '1234', + undefined + ); + const _id = calculateThresholdSignalUuid('1234', startedAt, ['source.ip'], '127.0.0.1'); + expect(transformedResults).toEqual({ + took: 10, + timed_out: false, + _shards: { + total: 10, + successful: 10, + failed: 0, + skipped: 0, + }, + results: { + hits: { + total: 1, + }, + }, + hits: { + total: 100, + max_score: 100, + hits: [ + { + _id, + _index: 'test', + _source: { + '@timestamp': '2020-04-20T21:27:45+0000', + threshold_result: { + terms: [ + { + field: 'source.ip', + value: '127.0.0.1', + }, + ], + cardinality: undefined, + count: 15, + }, + }, + }, + ], + }, + }); + }); + + it('should return transformed threshold results for pre-7.12 rules without threshold field', () => { + const threshold: Threshold = { + field: '', + value: 1, + }; + const startedAt = new Date('2020-12-17T16:27:00Z'); + const transformedResults = transformThresholdResultsToEcs( + { + ...sampleDocSearchResultsNoSortId('abcd'), + aggregations: { + threshold_0: { + buckets: [ + { + key: '', + doc_count: 15, + top_threshold_hits: { + hits: { + hits: [sampleDocNoSortId('abcd')], + }, + }, + }, + ], + }, + }, + }, + 'test', + startedAt, + undefined, + loggingSystemMock.createLogger(), + { + ...threshold, + field: normalizeThresholdField(threshold.field), + }, + '1234', + undefined + ); + const _id = calculateThresholdSignalUuid('1234', startedAt, [], ''); + expect(transformedResults).toEqual({ + took: 10, + timed_out: false, + _shards: { + total: 10, + successful: 10, + failed: 0, + skipped: 0, + }, + results: { + hits: { + total: 1, + }, + }, + hits: { + total: 100, + max_score: 100, + hits: [ + { + _id, + _index: 'test', + _source: { + '@timestamp': '2020-04-20T21:27:45+0000', + threshold_result: { + terms: [], + cardinality: undefined, + count: 15, + }, + }, + }, + ], + }, + }); + }); + + it('should return transformed threshold results', () => { + const threshold: ThresholdNormalized = { field: ['source.ip', 'host.name'], value: 1, - cardinality_field: 'destination.ip', - cardinality_value: 5, + cardinality: [ + { + field: 'destination.ip', + value: 5, + }, + ], }; const startedAt = new Date('2020-12-17T16:27:00Z'); const transformedResults = transformThresholdResultsToEcs( @@ -112,4 +269,87 @@ describe('transformThresholdResultsToEcs', () => { }, }); }); + + it('should return transformed threshold results without threshold fields', () => { + const threshold: ThresholdNormalized = { + field: [], + value: 1, + cardinality: [ + { + field: 'destination.ip', + value: 5, + }, + ], + }; + const startedAt = new Date('2020-12-17T16:27:00Z'); + const transformedResults = transformThresholdResultsToEcs( + { + ...sampleDocSearchResultsNoSortId('abcd'), + aggregations: { + threshold_0: { + buckets: [ + { + key: '', + doc_count: 15, + top_threshold_hits: { + hits: { + hits: [sampleDocNoSortId('abcd')], + }, + }, + cardinality_count: { + value: 7, + }, + }, + ], + }, + }, + }, + 'test', + startedAt, + undefined, + loggingSystemMock.createLogger(), + threshold, + '1234', + undefined + ); + const _id = calculateThresholdSignalUuid('1234', startedAt, [], ''); + expect(transformedResults).toEqual({ + took: 10, + timed_out: false, + _shards: { + total: 10, + successful: 10, + failed: 0, + skipped: 0, + }, + results: { + hits: { + total: 1, + }, + }, + hits: { + total: 100, + max_score: 100, + hits: [ + { + _id, + _index: 'test', + _source: { + '@timestamp': '2020-04-20T21:27:45+0000', + threshold_result: { + terms: [], + cardinality: [ + { + field: 'destination.ip', + value: 7, + }, + ], + count: 15, + }, + }, + }, + ], + }, + }); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts index 29fd189bb34f3..1c1b2bb18900a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts @@ -5,11 +5,12 @@ * 2.0. */ -import { get, isEmpty } from 'lodash/fp'; +import { get } from 'lodash/fp'; import set from 'set-value'; +import { normalizeThresholdField } from '../../../../common/detection_engine/utils'; import { - Threshold, + ThresholdNormalized, TimestampOverrideOrUndefined, } from '../../../../common/detection_engine/schemas/common/schemas'; import { Logger } from '../../../../../../../src/core/server'; @@ -56,59 +57,19 @@ const getTransformedHits = ( inputIndex: string, startedAt: Date, logger: Logger, - threshold: Threshold, + threshold: ThresholdNormalized, ruleId: string, filter: unknown, timestampOverride: TimestampOverrideOrUndefined ) => { - if (isEmpty(threshold.field)) { - const totalResults = - typeof results.hits.total === 'number' ? results.hits.total : results.hits.total.value; - - if (totalResults < threshold.value) { - return []; - } - - const hit = results.hits.hits[0]; - if (hit == null) { - logger.warn(`No hits returned, but totalResults >= threshold.value (${threshold.value})`); - return []; - } - const timestampArray = get(timestampOverride ?? '@timestamp', hit.fields); - if (timestampArray == null) { - return []; - } - const timestamp = timestampArray[0]; - if (typeof timestamp !== 'string') { - return []; - } - - const source = { - '@timestamp': timestamp, - threshold_result: { - terms: [ - { - value: ruleId, - }, - ], - count: totalResults, - }, - }; - - return [ - { - _index: inputIndex, - _id: calculateThresholdSignalUuid( - ruleId, - startedAt, - Array.isArray(threshold.field) ? threshold.field : [threshold.field] - ), - _source: source, - }, - ]; - } + const aggParts = threshold.field.length + ? results.aggregations && getThresholdAggregationParts(results.aggregations) + : { + field: null, + index: 0, + name: 'threshold_0', + }; - const aggParts = results.aggregations && getThresholdAggregationParts(results.aggregations); if (!aggParts) { return []; } @@ -119,7 +80,7 @@ const getTransformedHits = ( const nextLevelIdx = i + 1; const nextLevelAggParts = getThresholdAggregationParts(bucket, nextLevelIdx); if (nextLevelAggParts == null) { - throw new Error('Something went horribly wrong'); + throw new Error('Unable to parse aggregation.'); } const nextLevelPath = `['${nextLevelAggParts.name}']['buckets']`; const nextBuckets = get(nextLevelPath, bucket); @@ -132,7 +93,7 @@ const getTransformedHits = ( value: bucket.key, }, ...val.terms, - ], + ].filter((term) => term.field != null), cardinality: val.cardinality, topThresholdHits: val.topThresholdHits, docCount: val.docCount, @@ -146,13 +107,11 @@ const getTransformedHits = ( field, value: bucket.key, }, - ], - cardinality: !isEmpty(threshold.cardinality_field) + ].filter((term) => term.field != null), + cardinality: threshold.cardinality?.length ? [ { - field: Array.isArray(threshold.cardinality_field) - ? threshold.cardinality_field[0] - : threshold.cardinality_field!, + field: threshold.cardinality[0].field, value: bucket.cardinality_count!.value, }, ] @@ -208,7 +167,7 @@ const getTransformedHits = ( _id: calculateThresholdSignalUuid( ruleId, startedAt, - Array.isArray(threshold.field) ? threshold.field : [threshold.field], + threshold.field, bucket.terms.map((term) => term.value).join(',') ), _source: source, @@ -226,7 +185,7 @@ export const transformThresholdResultsToEcs = ( startedAt: Date, filter: unknown, logger: Logger, - threshold: Threshold, + threshold: ThresholdNormalized, ruleId: string, timestampOverride: TimestampOverrideOrUndefined ): SignalSearchResponse => { @@ -259,13 +218,17 @@ export const bulkCreateThresholdSignals = async ( params: BulkCreateThresholdSignalsParams ): Promise => { const thresholdResults = params.someResult; + const threshold = params.ruleParams.threshold!; const ecsResults = transformThresholdResultsToEcs( thresholdResults, params.inputIndexPattern.join(','), params.startedAt, params.filter, params.logger, - params.ruleParams.threshold!, + { + ...threshold, + field: normalizeThresholdField(threshold.field), + }, params.ruleParams.ruleId, params.timestampOverride ); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.test.ts new file mode 100644 index 0000000000000..6f7985fe52ecf --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.test.ts @@ -0,0 +1,445 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { alertsMock, AlertServicesMock } from '../../../../../alerts/server/mocks'; +import { getQueryFilter } from '../../../../common/detection_engine/get_query_filter'; +import { mockLogger } from './__mocks__/es_results'; +import { findThresholdSignals } from './find_threshold_signals'; +import { buildRuleMessageFactory } from './rule_messages'; +import * as single_search_after from './single_search_after'; + +const buildRuleMessage = buildRuleMessageFactory({ + id: 'fake id', + ruleId: 'fake rule id', + index: 'fakeindex', + name: 'fake name', +}); + +const queryFilter = getQueryFilter('', 'kuery', [], ['*'], []); +const mockSingleSearchAfter = jest.fn(); + +describe('findThresholdSignals', () => { + let mockService: AlertServicesMock; + + beforeEach(() => { + jest.clearAllMocks(); + jest.spyOn(single_search_after, 'singleSearchAfter').mockImplementation(mockSingleSearchAfter); + mockService = alertsMock.createAlertServices(); + }); + + it('should generate a threshold signal for pre-7.12 rules', async () => { + await findThresholdSignals({ + from: 'now-6m', + to: 'now', + inputIndexPattern: ['*'], + services: mockService, + logger: mockLogger, + filter: queryFilter, + threshold: { + field: 'host.name', + value: 100, + }, + buildRuleMessage, + timestampOverride: undefined, + }); + expect(mockSingleSearchAfter).toHaveBeenCalledWith( + expect.objectContaining({ + aggregations: { + 'threshold_0:host.name': { + terms: { + field: 'host.name', + min_doc_count: 100, + size: 10000, + }, + aggs: { + top_threshold_hits: { + top_hits: { + sort: [ + { + '@timestamp': { + order: 'desc', + }, + }, + ], + fields: [ + { + field: '*', + include_unmapped: true, + }, + ], + size: 1, + }, + }, + }, + }, + }, + }) + ); + }); + + it('should generate a signal for pre-7.12 rules with no threshold field', async () => { + await findThresholdSignals({ + from: 'now-6m', + to: 'now', + inputIndexPattern: ['*'], + services: mockService, + logger: mockLogger, + filter: queryFilter, + threshold: { + field: '', + value: 100, + }, + buildRuleMessage, + timestampOverride: undefined, + }); + expect(mockSingleSearchAfter).toHaveBeenCalledWith( + expect.objectContaining({ + aggregations: { + threshold_0: { + terms: { + script: { + source: '""', + lang: 'painless', + }, + min_doc_count: 100, + }, + aggs: { + top_threshold_hits: { + top_hits: { + sort: [ + { + '@timestamp': { + order: 'desc', + }, + }, + ], + fields: [ + { + field: '*', + include_unmapped: true, + }, + ], + size: 1, + }, + }, + }, + }, + }, + }) + ); + }); + + it('should generate a threshold signal query when only a value is provided', async () => { + await findThresholdSignals({ + from: 'now-6m', + to: 'now', + inputIndexPattern: ['*'], + services: mockService, + logger: mockLogger, + filter: queryFilter, + threshold: { + field: [], + value: 100, + }, + buildRuleMessage, + timestampOverride: undefined, + }); + expect(mockSingleSearchAfter).toHaveBeenCalledWith( + expect.objectContaining({ + aggregations: { + threshold_0: { + terms: { + script: { + source: '""', + lang: 'painless', + }, + min_doc_count: 100, + }, + aggs: { + top_threshold_hits: { + top_hits: { + sort: [ + { + '@timestamp': { + order: 'desc', + }, + }, + ], + fields: [ + { + field: '*', + include_unmapped: true, + }, + ], + size: 1, + }, + }, + }, + }, + }, + }) + ); + }); + + it('should generate a threshold signal query when a field and value are provided', async () => { + await findThresholdSignals({ + from: 'now-6m', + to: 'now', + inputIndexPattern: ['*'], + services: mockService, + logger: mockLogger, + filter: queryFilter, + threshold: { + field: ['host.name'], + value: 100, + }, + buildRuleMessage, + timestampOverride: undefined, + }); + expect(mockSingleSearchAfter).toHaveBeenCalledWith( + expect.objectContaining({ + aggregations: { + 'threshold_0:host.name': { + terms: { + field: 'host.name', + min_doc_count: 100, + size: 10000, + }, + aggs: { + top_threshold_hits: { + top_hits: { + sort: [ + { + '@timestamp': { + order: 'desc', + }, + }, + ], + fields: [ + { + field: '*', + include_unmapped: true, + }, + ], + size: 1, + }, + }, + }, + }, + }, + }) + ); + }); + + it('should generate a threshold signal query when multiple fields and a value are provided', async () => { + await findThresholdSignals({ + from: 'now-6m', + to: 'now', + inputIndexPattern: ['*'], + services: mockService, + logger: mockLogger, + filter: queryFilter, + threshold: { + field: ['host.name', 'user.name'], + value: 100, + }, + buildRuleMessage, + timestampOverride: undefined, + }); + expect(mockSingleSearchAfter).toHaveBeenCalledWith( + expect.objectContaining({ + aggregations: { + 'threshold_0:host.name': { + terms: { + field: 'host.name', + min_doc_count: 100, + size: 10000, + }, + aggs: { + 'threshold_1:user.name': { + terms: { + field: 'user.name', + min_doc_count: 100, + size: 10000, + }, + aggs: { + top_threshold_hits: { + top_hits: { + sort: [ + { + '@timestamp': { + order: 'desc', + }, + }, + ], + fields: [ + { + field: '*', + include_unmapped: true, + }, + ], + size: 1, + }, + }, + }, + }, + }, + }, + }, + }) + ); + }); + + it('should generate a threshold signal query when multiple fields, a value, and cardinality field/value are provided', async () => { + await findThresholdSignals({ + from: 'now-6m', + to: 'now', + inputIndexPattern: ['*'], + services: mockService, + logger: mockLogger, + filter: queryFilter, + threshold: { + field: ['host.name', 'user.name'], + value: 100, + cardinality: [ + { + field: 'destination.ip', + value: 2, + }, + ], + }, + buildRuleMessage, + timestampOverride: undefined, + }); + expect(mockSingleSearchAfter).toHaveBeenCalledWith( + expect.objectContaining({ + aggregations: { + 'threshold_0:host.name': { + terms: { + field: 'host.name', + min_doc_count: 100, + size: 10000, + }, + aggs: { + 'threshold_1:user.name': { + terms: { + field: 'user.name', + min_doc_count: 100, + size: 10000, + }, + aggs: { + cardinality_count: { + cardinality: { + field: 'destination.ip', + }, + }, + cardinality_check: { + bucket_selector: { + buckets_path: { + cardinalityCount: 'cardinality_count', + }, + script: 'params.cardinalityCount >= 2', + }, + }, + top_threshold_hits: { + top_hits: { + sort: [ + { + '@timestamp': { + order: 'desc', + }, + }, + ], + fields: [ + { + field: '*', + include_unmapped: true, + }, + ], + size: 1, + }, + }, + }, + }, + }, + }, + }, + }) + ); + }); + + it('should generate a threshold signal query when only a value and a cardinality field/value are provided', async () => { + await findThresholdSignals({ + from: 'now-6m', + to: 'now', + inputIndexPattern: ['*'], + services: mockService, + logger: mockLogger, + filter: queryFilter, + threshold: { + cardinality: [ + { + field: 'source.ip', + value: 5, + }, + ], + field: [], + value: 200, + }, + buildRuleMessage, + timestampOverride: undefined, + }); + expect(mockSingleSearchAfter).toHaveBeenCalledWith( + expect.objectContaining({ + aggregations: { + threshold_0: { + terms: { + script: { + source: '""', + lang: 'painless', + }, + min_doc_count: 200, + }, + aggs: { + cardinality_count: { + cardinality: { + field: 'source.ip', + }, + }, + cardinality_check: { + bucket_selector: { + buckets_path: { + cardinalityCount: 'cardinality_count', + }, + script: 'params.cardinalityCount >= 5', + }, + }, + top_threshold_hits: { + top_hits: { + sort: [ + { + '@timestamp': { + order: 'desc', + }, + }, + ], + fields: [ + { + field: '*', + include_unmapped: true, + }, + ], + size: 1, + }, + }, + }, + }, + }, + }) + ); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts index 7796346e9876d..8b446bba90f65 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts @@ -6,12 +6,12 @@ */ import { set } from '@elastic/safer-lodash-set'; -import { isEmpty } from 'lodash/fp'; import { Threshold, TimestampOverrideOrUndefined, } from '../../../../common/detection_engine/schemas/common/schemas'; +import { normalizeThresholdField } from '../../../../common/detection_engine/utils'; import { singleSearchAfter } from './single_search_after'; import { @@ -50,69 +50,98 @@ export const findThresholdSignals = async ({ searchDuration: string; searchErrors: string[]; }> => { - const thresholdFields = Array.isArray(threshold.field) ? threshold.field : [threshold.field]; + const topHitsAgg = { + top_hits: { + sort: [ + { + [timestampOverride ?? '@timestamp']: { + order: 'desc', + }, + }, + ], + fields: [ + { + field: '*', + include_unmapped: true, + }, + ], + size: 1, + }, + }; - const aggregations = - threshold && !isEmpty(threshold.field) - ? thresholdFields.reduce((acc, field, i) => { - const aggPath = [...Array(i + 1).keys()] - .map((j) => { - return `['threshold_${j}:${thresholdFields[j]}']`; - }) - .join(`['aggs']`); - set(acc, aggPath, { - terms: { - field, - min_doc_count: threshold.value, // not needed on parent agg, but can help narrow down result set - size: 10000, // max 10k buckets - }, - }); - if (i === threshold.field.length - 1) { - const topHitsAgg = { - top_hits: { - sort: [ - { - [timestampOverride ?? '@timestamp']: { - order: 'desc', - }, - }, - ], - fields: [ - { - field: '*', - include_unmapped: true, - }, - ], - size: 1, + const thresholdFields = normalizeThresholdField(threshold.field); + + const aggregations = thresholdFields.length + ? thresholdFields.reduce((acc, field, i) => { + const aggPath = [...Array(i + 1).keys()] + .map((j) => { + return `['threshold_${j}:${thresholdFields[j]}']`; + }) + .join(`['aggs']`); + set(acc, aggPath, { + terms: { + field, + min_doc_count: threshold.value, // not needed on parent agg, but can help narrow down result set + size: 10000, // max 10k buckets + }, + }); + if (i === (thresholdFields.length ?? 0) - 1) { + if (threshold.cardinality?.length) { + set(acc, `${aggPath}['aggs']`, { + top_threshold_hits: topHitsAgg, + cardinality_count: { + cardinality: { + field: threshold.cardinality[0].field, + }, }, - }; - // TODO: support case where threshold fields are not supplied, but cardinality is? - if (!isEmpty(threshold.cardinality_field)) { - set(acc, `${aggPath}['aggs']`, { - top_threshold_hits: topHitsAgg, - cardinality_count: { - cardinality: { - field: threshold.cardinality_field, + cardinality_check: { + bucket_selector: { + buckets_path: { + cardinalityCount: 'cardinality_count', }, + script: `params.cardinalityCount >= ${threshold.cardinality[0].value}`, // TODO: cardinality operator }, - cardinality_check: { - bucket_selector: { - buckets_path: { - cardinalityCount: 'cardinality_count', + }, + }); + } else { + set(acc, `${aggPath}['aggs']`, { + top_threshold_hits: topHitsAgg, + }); + } + } + return acc; + }, {}) + : { + threshold_0: { + terms: { + script: { + source: '""', + lang: 'painless', + }, + min_doc_count: threshold.value, + }, + aggs: { + top_threshold_hits: topHitsAgg, + ...(threshold.cardinality?.length + ? { + cardinality_count: { + cardinality: { + field: threshold.cardinality[0].field, }, - script: `params.cardinalityCount >= ${threshold.cardinality_value}`, // TODO: cardinality operator }, - }, - }); - } else { - set(acc, `${aggPath}['aggs']`, { - top_threshold_hits: topHitsAgg, - }); - } - } - return acc; - }, {}) - : {}; + cardinality_check: { + bucket_selector: { + buckets_path: { + cardinalityCount: 'cardinality_count', + }, + script: `params.cardinalityCount >= ${threshold.cardinality[0].value}`, // TODO: cardinality operator + }, + }, + } + : {}), + }, + }, + }; return singleSearchAfter({ aggregations, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.test.ts index 06c5fd38099bc..21db1e55b9810 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.test.ts @@ -103,4 +103,56 @@ describe('signal_params_schema', () => { const { falsePositives, ...withoutFalsePositives } = getSignalParamsSchemaMock(); expect(schema.validate(withoutFalsePositives).falsePositives).toEqual([]); }); + + test('threshold validates with `value` only', () => { + const schema = signalParamsSchema(); + const threshold = { + value: 200, + }; + const mock = { + ...getSignalParamsSchemaMock(), + threshold, + }; + expect(schema.validate(mock).threshold?.value).toEqual(200); + }); + + test('threshold does not validate without `value`', () => { + const schema = signalParamsSchema(); + const threshold = { + field: 'agent.id', + cardinality: [ + { + field: ['host.name'], + value: 5, + }, + ], + }; + const mock = { + ...getSignalParamsSchemaMock(), + threshold, + }; + expect(() => schema.validate(mock)).toThrow(); + }); + + test('threshold `cardinality` cannot currently be greater than length 1', () => { + const schema = signalParamsSchema(); + const threshold = { + value: 100, + cardinality: [ + { + field: 'host.name', + value: 5, + }, + { + field: 'user.name', + value: 5, + }, + ], + }; + const mock = { + ...getSignalParamsSchemaMock(), + threshold, + }; + expect(() => schema.validate(mock)).toThrow(); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts index 710a925fe315b..bfa452af0f3e9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts @@ -41,10 +41,19 @@ export const signalSchema = schema.object({ threat: schema.nullable(schema.arrayOf(schema.object({}, { unknowns: 'allow' }))), threshold: schema.maybe( schema.object({ + // Can be an empty string or empty array field: schema.nullable(schema.oneOf([schema.string(), schema.arrayOf(schema.string())])), + // Always required value: schema.number(), - cardinality_field: schema.nullable(schema.string()), // TODO: depends on `field` being defined? - cardinality_value: schema.nullable(schema.number()), + cardinality: schema.nullable( + schema.arrayOf( + schema.object({ + field: schema.string(), + value: schema.number(), + }), + { maxSize: 1 } + ) + ), }) ), timestampOverride: schema.nullable(schema.string()), diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 9c79b17b1e16d..94415f9b59e00 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -25,6 +25,7 @@ import { isEqlRule, isThreatMatchRule, hasLargeValueItem, + normalizeThresholdField, } from '../../../../common/detection_engine/utils'; import { parseScheduleDates } from '../../../../common/detection_engine/parse_schedule_dates'; import { SetupPlugins } from '../../../plugin'; @@ -381,10 +382,6 @@ export const signalRulesAlertType = ({ } const inputIndex = await getInputIndex(services, version, index); - const thresholdFields = Array.isArray(threshold.field) - ? threshold.field - : [threshold.field]; - const { filters: bucketFilters, searchErrors: previousSearchErrors, @@ -395,7 +392,7 @@ export const signalRulesAlertType = ({ services, logger, ruleId, - bucketByFields: thresholdFields, + bucketByFields: normalizeThresholdField(threshold.field), timestampOverride, buildRuleMessage, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_get_bucket_filters.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_get_bucket_filters.test.ts index ed9aa9a5ba698..97d7d96a50e12 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_get_bucket_filters.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_get_bucket_filters.test.ts @@ -6,7 +6,11 @@ */ import { alertsMock, AlertServicesMock } from '../../../../../alerts/server/mocks'; -import { mockLogger, sampleWrappedThresholdSignalHit } from './__mocks__/es_results'; +import { + mockLogger, + sampleWrappedThresholdSignalHit, + sampleWrappedLegacyThresholdSignalHit, +} from './__mocks__/es_results'; import { getThresholdBucketFilters } from './threshold_get_bucket_filters'; import { buildRuleMessageFactory } from './rule_messages'; @@ -82,4 +86,131 @@ describe('thresholdGetBucketFilters', () => { searchErrors: [], }); }); + + it('should generate filters for threshold signal detection based on pre-7.12 signals', async () => { + mockService.callCluster.mockResolvedValue({ + took: 10, + timed_out: false, + _shards: { + total: 10, + successful: 10, + failed: 0, + skipped: 0, + }, + hits: { + total: 1, + max_score: 100, + hits: [sampleWrappedLegacyThresholdSignalHit()], + }, + }); + const result = await getThresholdBucketFilters({ + from: 'now-6m', + to: 'now', + indexPattern: ['*'], + services: mockService, + logger: mockLogger, + ruleId: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9', + bucketByFields: ['host.name'], + timestampOverride: undefined, + buildRuleMessage, + }); + expect(result).toEqual({ + filters: [ + { + bool: { + must_not: [ + { + bool: { + filter: [ + { + range: { + '@timestamp': { + lte: '2021-02-16T17:37:34.275Z', + }, + }, + }, + { + term: { + 'host.name': 'a hostname', + }, + }, + ], + }, + }, + ], + }, + }, + ], + searchErrors: [], + }); + }); + + it('should generate filters for threshold signal detection with mixed pre-7.12 and post-7.12 signals', async () => { + const signalHit = sampleWrappedThresholdSignalHit(); + const wrappedSignalHit = { + ...signalHit, + _source: { + ...signalHit._source, + signal: { + ...signalHit._source.signal, + original_time: '2021-02-16T18:37:34.275Z', + }, + }, + }; + mockService.callCluster.mockResolvedValue({ + took: 10, + timed_out: false, + _shards: { + total: 10, + successful: 10, + failed: 0, + skipped: 0, + }, + hits: { + total: 1, + max_score: 100, + hits: [sampleWrappedLegacyThresholdSignalHit(), wrappedSignalHit], + }, + }); + const result = await getThresholdBucketFilters({ + from: 'now-6m', + to: 'now', + indexPattern: ['*'], + services: mockService, + logger: mockLogger, + ruleId: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9', + bucketByFields: ['host.name'], + timestampOverride: undefined, + buildRuleMessage, + }); + expect(result).toEqual({ + filters: [ + { + bool: { + must_not: [ + { + bool: { + filter: [ + { + range: { + '@timestamp': { + lte: '2021-02-16T18:37:34.275Z', + }, + }, + }, + { + term: { + 'host.name': 'a hostname', + }, + }, + ], + }, + }, + ], + }, + }, + ], + searchErrors: [], + }); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_get_bucket_filters.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_get_bucket_filters.ts index e1727c0361afc..3091a8d535034 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_get_bucket_filters.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_get_bucket_filters.ts @@ -74,8 +74,9 @@ export const getThresholdBucketFilters = async ({ if (signalTerms == null) { signalTerms = [ { - field: (((hit._source.rule as RulesSchema).threshold as unknown) as { field: string }) - .field, + field: (((hit._source.signal?.rule as RulesSchema).threshold as unknown) as { + field: string; + }).field, value: ((hit._source.signal?.threshold_result as unknown) as { value: string }).value, }, ];