Skip to content

Commit

Permalink
[Security Solution][Detections] Fix flaky indicator enrichment tests (#…
Browse files Browse the repository at this point in the history 
…94241)

* Make indicator enrichment tests order-independent

Due to the fact that we use named queries to determine matches, and the
fact that the order in which named queries are returned is undefined, we
cannot guarantee a consistent ordering of enrichments if a given event
matches multiple named queries.

Because the ordering is not in itself important to enrichment, in order
to assert the multi-match functionality we must make the assertions
order independent.

* PR feedback

* Since we're only looping for side effects, prefer forEach to map for
  more idiomatic FP.
  • Loading branch information
@rylnd
rylnd authored Mar 10, 2021
1 parent 5acf15d commit 5c352ca
Showing 1 changed file with 161 additions and 160 deletions.
321 changes: 161 additions & 160 deletions ...test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
* 2.0.
*/

import { isEqual } from 'lodash';
import expect from '@kbn/expect';

import { CreateRulesSchema } from '../../../../plugins/security_solution/common/detection_engine/schemas/request';
Expand All @@ -27,6 +28,18 @@ import {
import { getCreateThreatMatchRulesSchemaMock } from '../../../../plugins/security_solution/common/detection_engine/schemas/request/rule_schemas.mock';
import { getThreatMatchingSchemaPartialMock } from '../../../../plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks';

const format = (value: unknown): string => JSON.stringify(value, null, 2);

// Asserts that each expected value is included in the subject, independent of
// ordering. Uses _.isEqual for value comparison.
const assertContains = (subject: unknown[], expected: unknown[]) =>
expected.forEach((expectedValue) =>
expect(subject.some((value) => isEqual(value, expectedValue))).to.eql(
true,
`expected ${format(subject)} to contain ${format(expectedValue)}`
)
);

// eslint-disable-next-line import/no-default-export
export default ({ getService }: FtrProviderContext) => {
const supertest = getService('supertest');
Expand All @@ -35,8 +48,7 @@ export default ({ getService }: FtrProviderContext) => {
/**
* Specific api integration tests for threat matching rule type
*/
// FLAKY: https://github.com/elastic/kibana/issues/93152
describe.skip('create_threat_matching', () => {
describe('create_threat_matching', () => {
describe('validation errors', () => {
it('should give an error that the index must exist first if it does not exist before creating a rule', async () => {
const { body } = await supertest
Expand Down Expand Up @@ -383,40 +395,37 @@ export default ({ getService }: FtrProviderContext) => {
expect(signalsOpen.hits.hits.length).equal(1);

const { hits } = signalsOpen.hits;
const threats = hits.map((hit) => hit._source.threat);
expect(threats).to.eql([
const [threat] = hits.map((hit) => hit._source.threat) as Array<{ indicator: unknown[] }>;

assertContains(threat.indicator, [
{
indicator: [
{
description: 'this should match auditbeat/hosts on both port and ip',
first_seen: '2021-01-26T11:06:03.000Z',
ip: '45.115.45.3',
matched: {
atomic: '45.115.45.3',
id: '978785',
index: 'filebeat-8.0.0-2021.01.26-000001',
field: 'source.ip',
type: 'url',
},
port: 57324,
provider: 'geenensp',
type: 'url',
},
{
description: 'this should match auditbeat/hosts on ip',
first_seen: '2021-01-26T11:06:03.000Z',
ip: '45.115.45.3',
matched: {
atomic: '45.115.45.3',
id: '978787',
index: 'filebeat-8.0.0-2021.01.26-000001',
field: 'source.ip',
type: 'ip',
},
provider: 'other_provider',
type: 'ip',
},
],
description: 'this should match auditbeat/hosts on both port and ip',
first_seen: '2021-01-26T11:06:03.000Z',
ip: '45.115.45.3',
matched: {
atomic: '45.115.45.3',
id: '978785',
index: 'filebeat-8.0.0-2021.01.26-000001',
field: 'source.ip',
type: 'url',
},
port: 57324,
provider: 'geenensp',
type: 'url',
},
{
description: 'this should match auditbeat/hosts on ip',
first_seen: '2021-01-26T11:06:03.000Z',
ip: '45.115.45.3',
matched: {
atomic: '45.115.45.3',
id: '978787',
index: 'filebeat-8.0.0-2021.01.26-000001',
field: 'source.ip',
type: 'ip',
},
provider: 'other_provider',
type: 'ip',
},
]);
});
Expand Down Expand Up @@ -466,61 +475,57 @@ export default ({ getService }: FtrProviderContext) => {
expect(signalsOpen.hits.hits.length).equal(1);

const { hits } = signalsOpen.hits;
const threats = hits.map((hit) => hit._source.threat);
const [threat] = hits.map((hit) => hit._source.threat) as Array<{ indicator: unknown[] }>;

expect(threats).to.eql([
assertContains(threat.indicator, [
{
indicator: [
{
description: 'this should match auditbeat/hosts on both port and ip',
first_seen: '2021-01-26T11:06:03.000Z',
ip: '45.115.45.3',
matched: {
atomic: '45.115.45.3',
id: '978785',
index: 'filebeat-8.0.0-2021.01.26-000001',
field: 'source.ip',
type: 'url',
},
port: 57324,
provider: 'geenensp',
type: 'url',
},
// We do not merge matched indicators during enrichment, so in
// certain circumstances a given indicator document could appear
// multiple times in an enriched alert (albeit with different
// threat.indicator.matched data). That's the case with the
// first and third indicators matched, here.
{
description: 'this should match auditbeat/hosts on both port and ip',
first_seen: '2021-01-26T11:06:03.000Z',
ip: '45.115.45.3',
matched: {
atomic: 57324,
id: '978785',
index: 'filebeat-8.0.0-2021.01.26-000001',
field: 'source.port',
type: 'url',
},
port: 57324,
provider: 'geenensp',
type: 'url',
},
{
description: 'this should match auditbeat/hosts on ip',
first_seen: '2021-01-26T11:06:03.000Z',
ip: '45.115.45.3',
matched: {
atomic: '45.115.45.3',
id: '978787',
index: 'filebeat-8.0.0-2021.01.26-000001',
field: 'source.ip',
type: 'ip',
},
provider: 'other_provider',
type: 'ip',
},
],
description: 'this should match auditbeat/hosts on both port and ip',
first_seen: '2021-01-26T11:06:03.000Z',
ip: '45.115.45.3',
matched: {
atomic: '45.115.45.3',
id: '978785',
index: 'filebeat-8.0.0-2021.01.26-000001',
field: 'source.ip',
type: 'url',
},
port: 57324,
provider: 'geenensp',
type: 'url',
},
// We do not merge matched indicators during enrichment, so in
// certain circumstances a given indicator document could appear
// multiple times in an enriched alert (albeit with different
// threat.indicator.matched data). That's the case with the
// first and third indicators matched, here.
{
description: 'this should match auditbeat/hosts on both port and ip',
first_seen: '2021-01-26T11:06:03.000Z',
ip: '45.115.45.3',
matched: {
atomic: 57324,
id: '978785',
index: 'filebeat-8.0.0-2021.01.26-000001',
field: 'source.port',
type: 'url',
},
port: 57324,
provider: 'geenensp',
type: 'url',
},
{
description: 'this should match auditbeat/hosts on ip',
first_seen: '2021-01-26T11:06:03.000Z',
ip: '45.115.45.3',
matched: {
atomic: '45.115.45.3',
id: '978787',
index: 'filebeat-8.0.0-2021.01.26-000001',
field: 'source.ip',
type: 'ip',
},
provider: 'other_provider',
type: 'ip',
},
]);
});
Expand Down Expand Up @@ -575,81 +580,77 @@ export default ({ getService }: FtrProviderContext) => {
expect(signalsOpen.hits.hits.length).equal(2);

const { hits } = signalsOpen.hits;
const threats = hits.map((hit) => hit._source.threat);
expect(threats).to.eql([
const threats = hits.map((hit) => hit._source.threat) as Array<{ indicator: unknown[] }>;

assertContains(threats[0].indicator, [
{
indicator: [
{
description: "domain should match the auditbeat hosts' data's source.ip",
domain: '159.89.119.67',
first_seen: '2021-01-26T11:09:04.000Z',
matched: {
atomic: '159.89.119.67',
id: '978783',
index: 'filebeat-8.0.0-2021.01.26-000001',
field: 'destination.ip',
type: 'url',
},
provider: 'geenensp',
type: 'url',
url: {
full: 'http://159.89.119.67:59600/bin.sh',
scheme: 'http',
},
},
],
description: "domain should match the auditbeat hosts' data's source.ip",
domain: '159.89.119.67',
first_seen: '2021-01-26T11:09:04.000Z',
matched: {
atomic: '159.89.119.67',
id: '978783',
index: 'filebeat-8.0.0-2021.01.26-000001',
field: 'destination.ip',
type: 'url',
},
provider: 'geenensp',
type: 'url',
url: {
full: 'http://159.89.119.67:59600/bin.sh',
scheme: 'http',
},
},
]);

assertContains(threats[1].indicator, [
{
indicator: [
{
description: "domain should match the auditbeat hosts' data's source.ip",
domain: '159.89.119.67',
first_seen: '2021-01-26T11:09:04.000Z',
matched: {
atomic: '159.89.119.67',
id: '978783',
index: 'filebeat-8.0.0-2021.01.26-000001',
field: 'destination.ip',
type: 'url',
},
provider: 'geenensp',
type: 'url',
url: {
full: 'http://159.89.119.67:59600/bin.sh',
scheme: 'http',
},
},
{
description: 'this should match auditbeat/hosts on both port and ip',
first_seen: '2021-01-26T11:06:03.000Z',
ip: '45.115.45.3',
matched: {
atomic: '45.115.45.3',
id: '978785',
index: 'filebeat-8.0.0-2021.01.26-000001',
field: 'source.ip',
type: 'url',
},
port: 57324,
provider: 'geenensp',
type: 'url',
},
{
description: 'this should match auditbeat/hosts on both port and ip',
first_seen: '2021-01-26T11:06:03.000Z',
ip: '45.115.45.3',
matched: {
atomic: 57324,
id: '978785',
index: 'filebeat-8.0.0-2021.01.26-000001',
field: 'source.port',
type: 'url',
},
port: 57324,
provider: 'geenensp',
type: 'url',
},
],
description: "domain should match the auditbeat hosts' data's source.ip",
domain: '159.89.119.67',
first_seen: '2021-01-26T11:09:04.000Z',
matched: {
atomic: '159.89.119.67',
id: '978783',
index: 'filebeat-8.0.0-2021.01.26-000001',
field: 'destination.ip',
type: 'url',
},
provider: 'geenensp',
type: 'url',
url: {
full: 'http://159.89.119.67:59600/bin.sh',
scheme: 'http',
},
},
{
description: 'this should match auditbeat/hosts on both port and ip',
first_seen: '2021-01-26T11:06:03.000Z',
ip: '45.115.45.3',
matched: {
atomic: '45.115.45.3',
id: '978785',
index: 'filebeat-8.0.0-2021.01.26-000001',
field: 'source.ip',
type: 'url',
},
port: 57324,
provider: 'geenensp',
type: 'url',
},
{
description: 'this should match auditbeat/hosts on both port and ip',
first_seen: '2021-01-26T11:06:03.000Z',
ip: '45.115.45.3',
matched: {
atomic: 57324,
id: '978785',
index: 'filebeat-8.0.0-2021.01.26-000001',
field: 'source.port',
type: 'url',
},
port: 57324,
provider: 'geenensp',
type: 'url',
},
]);
});
Expand Down

0 comments on commit 5c352ca

Please sign in to comment.