From f5bd489c5ff9c676c4f861c42da6ea99ae350832 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 14 Feb 2024 17:46:40 -0500 Subject: [PATCH] [8.12] [Bug][Investigations] - Fix slow timeline queries (#176838) (#176956) # Backport This will backport the following commits from `main` to `8.12`: - [[Bug][Investigations] - Fix slow timeline queries (#176838)](https://github.com/elastic/kibana/pull/176838) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) Co-authored-by: Michael Olorunnisola --- .../api/search_strategy/timeline/eql.test.ts | 109 ++++++++++++++++++ .../timeline/events_all.test.ts | 76 ++++++++++++ .../timeline/events_details.test.ts | 55 +++++++++ .../timeline/events_last_event_time.test.ts | 69 +++++++++++ .../api/search_strategy/timeline/kpi.test.ts | 51 ++++++++ .../timeline/mocks/base_timeline_request.ts | 20 ++++ .../timeline/request_basic.test.ts | 53 +++++++++ .../search_strategy/timeline/request_basic.ts | 1 + 8 files changed, 434 insertions(+) create mode 100644 x-pack/plugins/timelines/common/api/search_strategy/timeline/eql.test.ts create mode 100644 x-pack/plugins/timelines/common/api/search_strategy/timeline/events_all.test.ts create mode 100644 x-pack/plugins/timelines/common/api/search_strategy/timeline/events_details.test.ts create mode 100644 x-pack/plugins/timelines/common/api/search_strategy/timeline/events_last_event_time.test.ts create mode 100644 x-pack/plugins/timelines/common/api/search_strategy/timeline/kpi.test.ts create mode 100644 x-pack/plugins/timelines/common/api/search_strategy/timeline/mocks/base_timeline_request.ts create mode 100644 x-pack/plugins/timelines/common/api/search_strategy/timeline/request_basic.test.ts diff --git a/x-pack/plugins/timelines/common/api/search_strategy/timeline/eql.test.ts b/x-pack/plugins/timelines/common/api/search_strategy/timeline/eql.test.ts new file mode 100644 index 0000000000000..914bb685cd5f2 --- /dev/null +++ b/x-pack/plugins/timelines/common/api/search_strategy/timeline/eql.test.ts @@ -0,0 +1,109 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { timelineEqlRequestOptionsSchema } from './eql'; +import { mockBaseTimelineRequest } from './mocks/base_timeline_request'; + +const mockEqlRequestOptions = { + ...mockBaseTimelineRequest, + filterQuery: 'sequence\n[any where true]\n[any where true]', + eventCategoryField: 'event.category', + tiebreakerField: '', + fieldRequested: [ + '@timestamp', + 'message', + 'event.category', + 'event.action', + 'host.name', + 'source.ip', + 'destination.ip', + 'user.name', + '@timestamp', + 'kibana.alert.workflow_status', + 'kibana.alert.workflow_tags', + 'kibana.alert.workflow_assignee_ids', + 'kibana.alert.group.id', + 'kibana.alert.original_time', + 'kibana.alert.building_block_type', + 'kibana.alert.rule.from', + 'kibana.alert.rule.name', + 'kibana.alert.rule.to', + 'kibana.alert.rule.uuid', + 'kibana.alert.rule.rule_id', + 'kibana.alert.rule.type', + 'kibana.alert.suppression.docs_count', + 'kibana.alert.original_event.kind', + 'kibana.alert.original_event.module', + 'file.path', + 'file.Ext.code_signature.subject_name', + 'file.Ext.code_signature.trusted', + 'file.hash.sha256', + 'host.os.family', + 'event.code', + 'process.entry_leader.entity_id', + ], + language: 'eql', + pagination: { + activePage: 0, + querySize: 25, + }, + runtimeMappings: {}, + size: 100, + sort: [ + { + direction: 'asc', + esTypes: ['date'], + field: '@timestamp', + type: 'date', + }, + ], + timerange: { + from: '2018-02-12T20:39:22.229Z', + interval: '12h', + to: '2024-02-13T20:39:22.229Z', + }, + timestampField: '@timestamp', +}; + +describe('timelineEqlRequestOptionsSchema', () => { + it('should correctly parse the last eql request object without unknown fields', () => { + expect(timelineEqlRequestOptionsSchema.parse(mockEqlRequestOptions)).toEqual( + mockEqlRequestOptions + ); + }); + + it('should correctly parse the last eql request object and remove unknown fields', () => { + const invalidEqlRequest = { + ...mockEqlRequestOptions, + unknownField: 'should-be-removed', + }; + expect(timelineEqlRequestOptionsSchema.parse(invalidEqlRequest)).toEqual(mockEqlRequestOptions); + }); + + it('should correctly error if an incorrect field type is provided for a schema key', () => { + const invalidEqlRequest = { + ...mockEqlRequestOptions, + fieldRequested: 123, + }; + + expect(() => { + timelineEqlRequestOptionsSchema.parse(invalidEqlRequest); + }).toThrowErrorMatchingInlineSnapshot(` + "[ + { + \\"code\\": \\"invalid_type\\", + \\"expected\\": \\"array\\", + \\"received\\": \\"number\\", + \\"path\\": [ + \\"fieldRequested\\" + ], + \\"message\\": \\"Expected array, received number\\" + } + ]" + `); + }); +}); diff --git a/x-pack/plugins/timelines/common/api/search_strategy/timeline/events_all.test.ts b/x-pack/plugins/timelines/common/api/search_strategy/timeline/events_all.test.ts new file mode 100644 index 0000000000000..1c69e529c66f7 --- /dev/null +++ b/x-pack/plugins/timelines/common/api/search_strategy/timeline/events_all.test.ts @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { timelineEventsAllSchema } from './events_all'; +import { mockBaseTimelineRequest } from './mocks/base_timeline_request'; + +const mockEventsAllRequest = { + ...mockBaseTimelineRequest, + factoryQueryType: 'eventsAll', + excludeEcsData: false, + pagination: { activePage: 0, querySize: 25 }, + fieldRequested: [ + '@timestamp', + '_index', + 'message', + 'host.name', + 'event.module', + 'agent.type', + 'event.dataset', + 'event.action', + 'user.name', + 'source.ip', + 'destination.ip', + ], + sort: [ + { + field: '@timestamp', + type: 'date', + direction: 'desc', + esTypes: [], + }, + ], + fields: [], + language: 'kuery', +}; + +describe('timelineEventsAllSchema', () => { + it('should correctly parse the events request object', () => { + expect(timelineEventsAllSchema.parse(mockEventsAllRequest)).toEqual(mockEventsAllRequest); + }); + + it('should correctly parse the events request object and remove unknown fields', () => { + const invalidEventsRequest = { + ...mockEventsAllRequest, + unknownField: 'shouldBeRemoved', + }; + expect(timelineEventsAllSchema.parse(invalidEventsRequest)).toEqual(mockEventsAllRequest); + }); + + it('should correctly error if an incorrect field type is provided for a schema key', () => { + const invalidEventsRequest = { + ...mockEventsAllRequest, + excludeEcsData: 'notABoolean', + }; + + expect(() => { + timelineEventsAllSchema.parse(invalidEventsRequest); + }).toThrowErrorMatchingInlineSnapshot(` + "[ + { + \\"code\\": \\"invalid_type\\", + \\"expected\\": \\"boolean\\", + \\"received\\": \\"string\\", + \\"path\\": [ + \\"excludeEcsData\\" + ], + \\"message\\": \\"Expected boolean, received string\\" + } + ]" + `); + }); +}); diff --git a/x-pack/plugins/timelines/common/api/search_strategy/timeline/events_details.test.ts b/x-pack/plugins/timelines/common/api/search_strategy/timeline/events_details.test.ts new file mode 100644 index 0000000000000..57adc28cf8434 --- /dev/null +++ b/x-pack/plugins/timelines/common/api/search_strategy/timeline/events_details.test.ts @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { timelineEventsDetailsSchema } from './events_details'; + +const mockEventsDetails = { + entityType: 'events', + indexName: 'test-large-index', + eventId: 'enfXnY0Byt9Ce9tO1aWh', + factoryQueryType: 'eventsDetails', + runtimeMappings: {}, +}; + +describe('timelineEventsDetailsSchema', () => { + it('should correctly parse the event details request schema', () => { + expect(timelineEventsDetailsSchema.parse(mockEventsDetails)).toEqual(mockEventsDetails); + }); + + it('should correctly parse the event details request schema and remove unknown fields', () => { + const invalidEventsDetailsRequest = { + ...mockEventsDetails, + unknownField: 'should-be-removed', + }; + expect(timelineEventsDetailsSchema.parse(invalidEventsDetailsRequest)).toEqual( + mockEventsDetails + ); + }); + + it('should correctly error if an incorrect field type is provided for a schema key', () => { + const invalidEventsDetailsRequest = { + ...mockEventsDetails, + indexName: 123, + }; + + expect(() => { + timelineEventsDetailsSchema.parse(invalidEventsDetailsRequest); + }).toThrowErrorMatchingInlineSnapshot(` + "[ + { + \\"code\\": \\"invalid_type\\", + \\"expected\\": \\"string\\", + \\"received\\": \\"number\\", + \\"path\\": [ + \\"indexName\\" + ], + \\"message\\": \\"Expected string, received number\\" + } + ]" + `); + }); +}); diff --git a/x-pack/plugins/timelines/common/api/search_strategy/timeline/events_last_event_time.test.ts b/x-pack/plugins/timelines/common/api/search_strategy/timeline/events_last_event_time.test.ts new file mode 100644 index 0000000000000..1ccacb265416e --- /dev/null +++ b/x-pack/plugins/timelines/common/api/search_strategy/timeline/events_last_event_time.test.ts @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { timelineEventsLastEventTimeRequestSchema } from './events_last_event_time'; +import { mockBaseTimelineRequest } from './mocks/base_timeline_request'; + +const mockEventsLastEventTimeRequest = { + ...mockBaseTimelineRequest, + // Remove fields that are omitted in the schema + runtimeMappings: undefined, + filterQuery: undefined, + timerange: undefined, + // Add eventsLastEventTime specific fields + factoryQueryType: 'eventsLastEventTime', + indexKey: 'hosts', + details: {}, +}; + +describe('timelineEventsLastEventTimeRequestSchema', () => { + it('should correctly parse the last event time request object without unknown fields', () => { + expect(timelineEventsLastEventTimeRequestSchema.parse(mockEventsLastEventTimeRequest)).toEqual( + mockEventsLastEventTimeRequest + ); + }); + + it('should correctly parse the last event time request object and remove unknown fields', () => { + const invalidEventsDetailsRequest = { + ...mockEventsLastEventTimeRequest, + unknownField: 'should-be-removed', + }; + expect(timelineEventsLastEventTimeRequestSchema.parse(invalidEventsDetailsRequest)).toEqual( + mockEventsLastEventTimeRequest + ); + }); + + it('should correctly error if an incorrect field type is provided for a schema key', () => { + const invalidEventsDetailsRequest = { + ...mockEventsLastEventTimeRequest, + indexKey: 'unknown-key', + }; + + expect(() => { + timelineEventsLastEventTimeRequestSchema.parse(invalidEventsDetailsRequest); + }).toThrowErrorMatchingInlineSnapshot(` + "[ + { + \\"received\\": \\"unknown-key\\", + \\"code\\": \\"invalid_enum_value\\", + \\"options\\": [ + \\"hostDetails\\", + \\"hosts\\", + \\"users\\", + \\"userDetails\\", + \\"ipDetails\\", + \\"network\\" + ], + \\"path\\": [ + \\"indexKey\\" + ], + \\"message\\": \\"Invalid enum value. Expected 'hostDetails' | 'hosts' | 'users' | 'userDetails' | 'ipDetails' | 'network', received 'unknown-key'\\" + } + ]" + `); + }); +}); diff --git a/x-pack/plugins/timelines/common/api/search_strategy/timeline/kpi.test.ts b/x-pack/plugins/timelines/common/api/search_strategy/timeline/kpi.test.ts new file mode 100644 index 0000000000000..ade3b954e9210 --- /dev/null +++ b/x-pack/plugins/timelines/common/api/search_strategy/timeline/kpi.test.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { timelineKpiRequestOptionsSchema } from './kpi'; +import { mockBaseTimelineRequest } from './mocks/base_timeline_request'; + +const mockKpiRequest = { + ...mockBaseTimelineRequest, + factoryQueryType: 'eventsKpi', +}; + +describe('timelineKpiRequestOptionsSchema', () => { + it('should correctly parse the events kpi request object', () => { + expect(timelineKpiRequestOptionsSchema.parse(mockKpiRequest)).toEqual(mockKpiRequest); + }); + + it('should correctly parse the events kpi request object and remove unknown fields', () => { + const invalidKpiRequest = { + ...mockKpiRequest, + unknownField: 'shouldBeRemoved', + }; + expect(timelineKpiRequestOptionsSchema.parse(invalidKpiRequest)).toEqual(mockKpiRequest); + }); + + it('should correctly error if an incorrect field type is provided for a schema key', () => { + const invalidKpiRequest = { + ...mockKpiRequest, + factoryQueryType: 'someOtherType', + }; + + expect(() => { + timelineKpiRequestOptionsSchema.parse(invalidKpiRequest); + }).toThrowErrorMatchingInlineSnapshot(` + "[ + { + \\"received\\": \\"someOtherType\\", + \\"code\\": \\"invalid_literal\\", + \\"expected\\": \\"eventsKpi\\", + \\"path\\": [ + \\"factoryQueryType\\" + ], + \\"message\\": \\"Invalid literal value, expected \\\\\\"eventsKpi\\\\\\"\\" + } + ]" + `); + }); +}); diff --git a/x-pack/plugins/timelines/common/api/search_strategy/timeline/mocks/base_timeline_request.ts b/x-pack/plugins/timelines/common/api/search_strategy/timeline/mocks/base_timeline_request.ts new file mode 100644 index 0000000000000..c1ab5fc35b641 --- /dev/null +++ b/x-pack/plugins/timelines/common/api/search_strategy/timeline/mocks/base_timeline_request.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const mockBaseTimelineRequest = { + id: 'Fnh1dVQ4SDRTUldtRXpUcDEwZXliWHcdZXdlWVBFWkVSWHVIdzY4a19JbFRvUTozMzgzNzk=', + defaultIndex: ['*-large-index'], + filterQuery: + '{"bool":{"must":[],"filter":[{"bool":{"should":[{"exists":{"field":"host.name"}}],"minimum_should_match":1}},{"range":{"@timestamp":{"gte":"2019-02-13T15:39:10.392Z","lt":"2024-02-14T04:59:59.999Z","format":"strict_date_optional_time"}}}],"should":[],"must_not":[]}}', + runtimeMappings: {}, + timerange: { + interval: '12h', + from: '2019-02-13T15:39:10.392Z', + to: '2024-02-14T04:59:59.999Z', + }, + entityType: 'events', +}; diff --git a/x-pack/plugins/timelines/common/api/search_strategy/timeline/request_basic.test.ts b/x-pack/plugins/timelines/common/api/search_strategy/timeline/request_basic.test.ts new file mode 100644 index 0000000000000..2a6b4c59f95e6 --- /dev/null +++ b/x-pack/plugins/timelines/common/api/search_strategy/timeline/request_basic.test.ts @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { timelineRequestBasicOptionsSchema } from './request_basic'; +import { mockBaseTimelineRequest } from './mocks/base_timeline_request'; + +describe('timelineRequestBasicOptionsSchema', () => { + it('should correctly parse the base timeline request object', () => { + expect(timelineRequestBasicOptionsSchema.parse(mockBaseTimelineRequest)).toEqual( + mockBaseTimelineRequest + ); + }); + + it('should correctly parse the base timeline request object and remove unknown fields', () => { + const invalidBaseTimelineRequest = { + ...mockBaseTimelineRequest, + iAmNotAllowed: 'butWhy?', + }; + expect(timelineRequestBasicOptionsSchema.parse(invalidBaseTimelineRequest)).toEqual( + mockBaseTimelineRequest + ); + }); + + it('should correctly error if an incorrect field type is provided for a schema key', () => { + const invalidBaseTimelineRequest = { + ...mockBaseTimelineRequest, + entityType: 'notAValidEntityType', + }; + + expect(() => { + timelineRequestBasicOptionsSchema.parse(invalidBaseTimelineRequest); + }).toThrowErrorMatchingInlineSnapshot(` + "[ + { + \\"received\\": \\"notAValidEntityType\\", + \\"code\\": \\"invalid_enum_value\\", + \\"options\\": [ + \\"events\\", + \\"sessions\\" + ], + \\"path\\": [ + \\"entityType\\" + ], + \\"message\\": \\"Invalid enum value. Expected 'events' | 'sessions', received 'notAValidEntityType'\\" + } + ]" + `); + }); +}); diff --git a/x-pack/plugins/timelines/common/api/search_strategy/timeline/request_basic.ts b/x-pack/plugins/timelines/common/api/search_strategy/timeline/request_basic.ts index 5e8ea1caaa0fb..c9c3145833572 100644 --- a/x-pack/plugins/timelines/common/api/search_strategy/timeline/request_basic.ts +++ b/x-pack/plugins/timelines/common/api/search_strategy/timeline/request_basic.ts @@ -12,6 +12,7 @@ import { timerange } from '../model/timerange'; export const timelineRequestBasicOptionsSchema = z.object({ indexType: z.string().optional(), + id: z.string().optional(), timerange: timerange.optional(), filterQuery, defaultIndex: z.array(z.string()).optional(),