From 516d28f6450934c915b25d0886f698dc53db5c41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Wed, 28 Oct 2020 12:53:02 +0100 Subject: [PATCH] [Logs UI] Transmit and render array field values in log entries (#81385) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Alejandro Fernández Gómez --- .../common/http_api/log_entries/entries.ts | 5 +- .../infra/common/http_api/log_entries/item.ts | 2 +- x-pack/plugins/infra/common/typed_json.ts | 22 +- .../log_entry_actions_menu.test.tsx | 18 +- .../log_entry_actions_menu.tsx | 15 +- .../log_entry_flyout/log_entry_flyout.tsx | 6 +- .../logging/log_text_stream/field_value.tsx | 66 + .../log_text_stream/log_entry_column.tsx | 4 +- .../log_entry_field_column.test.tsx | 90 +- .../log_entry_field_column.tsx | 56 +- .../log_entry_message_column.test.tsx | 82 + .../log_entry_message_column.tsx | 55 +- .../category_example_message.tsx | 4 +- .../sections/anomalies/log_entry_example.tsx | 4 +- .../lib/adapters/framework/adapter_types.ts | 6 +- .../log_entries/kibana_log_entries_adapter.ts | 41 +- .../builtin_rules/filebeat_apache2.test.ts | 467 ++--- .../builtin_rules/filebeat_apache2.ts | 2 +- .../builtin_rules/filebeat_auditd.test.ts | 808 ++++++--- .../builtin_rules/filebeat_auditd.ts | 56 +- .../builtin_rules/filebeat_haproxy.test.ts | 1596 +++++++++-------- .../builtin_rules/filebeat_icinga.test.ts | 240 +-- .../builtin_rules/filebeat_iis.test.ts | 1074 +++++------ .../builtin_rules/filebeat_kafka.test.ts | 84 +- .../builtin_rules/filebeat_logstash.test.ts | 376 ++-- .../builtin_rules/filebeat_logstash.ts | 4 +- .../builtin_rules/filebeat_mongodb.test.ts | 65 +- .../builtin_rules/filebeat_mysql.test.ts | 267 +-- .../builtin_rules/filebeat_nginx.test.ts | 475 ++--- .../builtin_rules/filebeat_osquery.test.ts | 186 +- .../builtin_rules/filebeat_osquery.ts | 2 +- .../builtin_rules/filebeat_traefik.test.ts | 221 +-- .../builtin_rules/generic.test.ts | 234 +-- .../builtin_rules/helpers.ts | 7 + ...document_source_to_log_item_fields.test.ts | 46 +- ...vert_document_source_to_log_item_fields.ts | 40 +- .../log_entries_domain/log_entries_domain.ts | 20 +- .../lib/domains/log_entries_domain/message.ts | 109 +- .../domains/log_entries_domain/rule_types.ts | 21 +- .../apis/metrics_ui/log_item.ts | 52 +- 40 files changed, 3963 insertions(+), 2965 deletions(-) create mode 100644 x-pack/plugins/infra/public/components/logging/log_text_stream/field_value.tsx create mode 100644 x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_message_column.test.tsx diff --git a/x-pack/plugins/infra/common/http_api/log_entries/entries.ts b/x-pack/plugins/infra/common/http_api/log_entries/entries.ts index 5014aeb52a3f7..d38b4690fed71 100644 --- a/x-pack/plugins/infra/common/http_api/log_entries/entries.ts +++ b/x-pack/plugins/infra/common/http_api/log_entries/entries.ts @@ -5,6 +5,7 @@ */ import * as rt from 'io-ts'; +import { jsonArrayRT } from '../../typed_json'; import { logEntriesCursorRT } from './common'; export const LOG_ENTRIES_PATH = '/api/log_entries/entries'; @@ -54,7 +55,7 @@ export const logMessageConstantPartRT = rt.type({ }); export const logMessageFieldPartRT = rt.type({ field: rt.string, - value: rt.unknown, + value: jsonArrayRT, highlights: rt.array(rt.string), }); @@ -64,7 +65,7 @@ export const logTimestampColumnRT = rt.type({ columnId: rt.string, timestamp: rt export const logFieldColumnRT = rt.type({ columnId: rt.string, field: rt.string, - value: rt.unknown, + value: jsonArrayRT, highlights: rt.array(rt.string), }); export const logMessageColumnRT = rt.type({ diff --git a/x-pack/plugins/infra/common/http_api/log_entries/item.ts b/x-pack/plugins/infra/common/http_api/log_entries/item.ts index 02335d68402c0..5f9457b8228ac 100644 --- a/x-pack/plugins/infra/common/http_api/log_entries/item.ts +++ b/x-pack/plugins/infra/common/http_api/log_entries/item.ts @@ -16,7 +16,7 @@ export const logEntriesItemRequestRT = rt.type({ export type LogEntriesItemRequest = rt.TypeOf; -const logEntriesItemFieldRT = rt.type({ field: rt.string, value: rt.string }); +const logEntriesItemFieldRT = rt.type({ field: rt.string, value: rt.array(rt.string) }); const logEntriesItemRT = rt.type({ id: rt.string, index: rt.string, diff --git a/x-pack/plugins/infra/common/typed_json.ts b/x-pack/plugins/infra/common/typed_json.ts index 98b5456fe44b8..0ff9e42942ef2 100644 --- a/x-pack/plugins/infra/common/typed_json.ts +++ b/x-pack/plugins/infra/common/typed_json.ts @@ -4,11 +4,21 @@ * you may not use this file except in compliance with the Elastic License. */ -export type JsonValue = null | boolean | number | string | JsonObject | JsonArray; +import * as rt from 'io-ts'; +import { JsonArray, JsonObject, JsonValue } from '../../../../src/plugins/kibana_utils/common'; -// eslint-disable-next-line @typescript-eslint/no-empty-interface -export interface JsonArray extends Array {} +export const jsonScalarRT = rt.union([rt.null, rt.boolean, rt.number, rt.string]); -export interface JsonObject { - [key: string]: JsonValue; -} +export const jsonValueRT: rt.Type = rt.recursion('JsonValue', () => + rt.union([jsonScalarRT, jsonArrayRT, jsonObjectRT]) +); + +export const jsonArrayRT: rt.Type = rt.recursion('JsonArray', () => + rt.array(jsonValueRT) +); + +export const jsonObjectRT: rt.Type = rt.recursion('JsonObject', () => + rt.record(rt.string, jsonValueRT) +); + +export { JsonValue, JsonArray, JsonObject }; diff --git a/x-pack/plugins/infra/public/components/logging/log_entry_flyout/log_entry_actions_menu.test.tsx b/x-pack/plugins/infra/public/components/logging/log_entry_flyout/log_entry_actions_menu.test.tsx index b4fa6b8800fba..77154474077c8 100644 --- a/x-pack/plugins/infra/public/components/logging/log_entry_flyout/log_entry_actions_menu.test.tsx +++ b/x-pack/plugins/infra/public/components/logging/log_entry_flyout/log_entry_actions_menu.test.tsx @@ -29,7 +29,7 @@ describe('LogEntryActionsMenu component', () => { { { { { { { .filter(({ field, value }) => value != null && UPTIME_FIELDS.includes(field)) .reduce((acc, fieldItem) => { const { field, value } = fieldItem; - try { - const parsedValue = decodeOrThrow(rt.array(rt.string))(JSON.parse(value)); - return acc.concat(parsedValue.map((val) => `${field}:${val}`)); - } catch (e) { - return acc.concat([`${field}:${value}`]); - } + return acc.concat(value.map((val) => `${field}:${val}`)); }, []); if (searchExpressions.length === 0) { @@ -119,7 +112,7 @@ const getUptimeLink = (logItem: LogEntriesItem): LinkDescriptor | undefined => { const getAPMLink = (logItem: LogEntriesItem): LinkDescriptor | undefined => { const traceIdEntry = logItem.fields.find( - ({ field, value }) => value != null && field === 'trace.id' + ({ field, value }) => value[0] != null && field === 'trace.id' ); if (!traceIdEntry) { @@ -127,7 +120,7 @@ const getAPMLink = (logItem: LogEntriesItem): LinkDescriptor | undefined => { } const timestampField = logItem.fields.find(({ field }) => field === '@timestamp'); - const timestamp = timestampField ? timestampField.value : null; + const timestamp = timestampField ? timestampField.value[0] : null; const { rangeFrom, rangeTo } = timestamp ? (() => { const from = new Date(timestamp); @@ -142,6 +135,6 @@ const getAPMLink = (logItem: LogEntriesItem): LinkDescriptor | undefined => { return { app: 'apm', - hash: getTraceUrl({ traceId: traceIdEntry.value, rangeFrom, rangeTo }), + hash: getTraceUrl({ traceId: traceIdEntry.value[0], rangeFrom, rangeTo }), }; }; diff --git a/x-pack/plugins/infra/public/components/logging/log_entry_flyout/log_entry_flyout.tsx b/x-pack/plugins/infra/public/components/logging/log_entry_flyout/log_entry_flyout.tsx index 76ffada510e51..b07d8c9dce23c 100644 --- a/x-pack/plugins/infra/public/components/logging/log_entry_flyout/log_entry_flyout.tsx +++ b/x-pack/plugins/infra/public/components/logging/log_entry_flyout/log_entry_flyout.tsx @@ -94,7 +94,7 @@ export const LogEntryFlyout = ({ onClick={createFilterHandler(item)} /> - {item.value} + {formatValue(item.value)} ), }, @@ -147,3 +147,7 @@ export const InfraFlyoutLoadingPanel = euiStyled.div` bottom: 0; left: 0; `; + +function formatValue(value: string[]) { + return value.length > 1 ? value.join(', ') : value[0]; +} diff --git a/x-pack/plugins/infra/public/components/logging/log_text_stream/field_value.tsx b/x-pack/plugins/infra/public/components/logging/log_text_stream/field_value.tsx new file mode 100644 index 0000000000000..7caf34c0cd6b7 --- /dev/null +++ b/x-pack/plugins/infra/public/components/logging/log_text_stream/field_value.tsx @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import stringify from 'json-stable-stringify'; +import React from 'react'; +import { euiStyled } from '../../../../../observability/public'; +import { JsonArray, JsonValue } from '../../../../common/typed_json'; +import { ActiveHighlightMarker, highlightFieldValue, HighlightMarker } from './highlighting'; + +export const FieldValue: React.FC<{ + highlightTerms: string[]; + isActiveHighlight: boolean; + value: JsonArray; +}> = React.memo(({ highlightTerms, isActiveHighlight, value }) => { + if (value.length === 1) { + return ( + <> + {highlightFieldValue( + formatValue(value[0]), + highlightTerms, + isActiveHighlight ? ActiveHighlightMarker : HighlightMarker + )} + + ); + } else if (value.length > 1) { + return ( +
    + {value.map((entry, i) => ( + + {highlightFieldValue( + formatValue(entry), + highlightTerms, + isActiveHighlight ? ActiveHighlightMarker : HighlightMarker + )} + + ))} +
+ ); + } + + return null; +}); + +const formatValue = (value: JsonValue): string => { + if (typeof value === 'string') { + return value; + } + + return stringify(value); +}; + +const CommaSeparatedLi = euiStyled.li` + display: inline; + &:not(:last-child) { + margin-right: 1ex; + &::after { + content: ','; + } + } +`; diff --git a/x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_column.tsx b/x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_column.tsx index 51488f088e75a..9b039b7f2f3ba 100644 --- a/x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_column.tsx +++ b/x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_column.tsx @@ -39,7 +39,9 @@ export const LogEntryColumn = euiStyled.div.attrs(() => ({ overflow: hidden; `; -export const LogEntryColumnContent = euiStyled.div` +export const LogEntryColumnContent = euiStyled.div.attrs({ + 'data-test-subj': 'LogEntryColumnContent', +})` flex: 1 0 0%; padding: 2px ${COLUMN_PADDING}px; `; diff --git a/x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_field_column.test.tsx b/x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_field_column.test.tsx index d6068b6e60992..5813f08497a74 100644 --- a/x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_field_column.test.tsx +++ b/x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_field_column.test.tsx @@ -4,85 +4,113 @@ * you may not use this file except in compliance with the Elastic License. */ -import { mount } from 'enzyme'; +import { render } from '@testing-library/react'; import React from 'react'; - import { EuiThemeProvider } from '../../../../../observability/public'; +import { LogFieldColumn } from '../../../../common/http_api'; import { LogEntryFieldColumn } from './log_entry_field_column'; -import { LogColumn } from '../../../../common/http_api'; describe('LogEntryFieldColumn', () => { - it('should output a
    when displaying an Array of values', () => { - const column: LogColumn = { + it('renders a single value without a wrapping list', () => { + const column: LogFieldColumn = { + columnId: 'TEST_COLUMN', + field: 'TEST_FIELD', + value: ['a'], + highlights: [], + }; + + const renderResult = render( + , + { wrapper: EuiThemeProvider } + ); + + expect(renderResult.getByTestId('LogEntryColumnContent')).toHaveTextContent(/^a$/); + expect(renderResult.queryByTestId('LogEntryFieldValues')).toBe(null); + }); + + it('renders an array of values as a list', () => { + const column: LogFieldColumn = { columnId: 'TEST_COLUMN', field: 'TEST_FIELD', value: ['a', 'b', 'c'], highlights: [], }; - const component = mount( + const renderResult = render( , - { wrappingComponent: EuiThemeProvider } as any // https://github.com/DefinitelyTyped/DefinitelyTyped/issues/36075 + { wrapper: EuiThemeProvider } ); - expect(component.exists('ul')).toBe(true); - expect( - component.containsAllMatchingElements([ -
  • a
    • , -
  • b
    • , -
  • c
    • , - ]) - ).toBe(true); + expect(renderResult.getByTestId('LogEntryFieldValues')).not.toBeEmptyDOMElement(); + expect(renderResult.getByTestId('LogEntryFieldValue-0')).toHaveTextContent('a'); + expect(renderResult.getByTestId('LogEntryFieldValue-1')).toHaveTextContent('b'); + expect(renderResult.getByTestId('LogEntryFieldValue-2')).toHaveTextContent('c'); }); - it('should output a text representation of a passed complex value', () => { - const column: LogColumn = { + it('renders a text representation of a single complex object', () => { + const column: LogFieldColumn = { columnId: 'TEST_COLUMN', field: 'TEST_FIELD', - value: { - lat: 1, - lon: 2, - }, + value: [ + { + lat: 1, + lon: 2, + }, + ], highlights: [], }; - const component = mount( + const renderResult = render( , - { wrappingComponent: EuiThemeProvider } as any // https://github.com/DefinitelyTyped/DefinitelyTyped/issues/36075 + { wrapper: EuiThemeProvider } ); - expect(component.text()).toEqual('{"lat":1,"lon":2}'); + expect(renderResult.getByTestId('LogEntryColumnContent')).toHaveTextContent( + '{"lat":1,"lon":2}' + ); }); - it('should output just text when passed a non-Array', () => { - const column: LogColumn = { + it('renders text representations of a multiple complex objects', () => { + const column: LogFieldColumn = { columnId: 'TEST_COLUMN', field: 'TEST_FIELD', - value: 'foo', + value: [ + { + lat: 1, + lon: 2, + }, + [3, 4], + ], highlights: [], }; - const component = mount( + const renderResult = render( , - { wrappingComponent: EuiThemeProvider } as any // https://github.com/DefinitelyTyped/DefinitelyTyped/issues/36075 + { wrapper: EuiThemeProvider } ); - expect(component.exists('ul')).toBe(false); - expect(component.text()).toEqual('foo'); + expect(renderResult.getByTestId('LogEntryFieldValues')).not.toBeEmptyDOMElement(); + expect(renderResult.getByTestId('LogEntryFieldValue-0')).toHaveTextContent('{"lat":1,"lon":2}'); + expect(renderResult.getByTestId('LogEntryFieldValue-1')).toHaveTextContent('[3,4]'); }); }); diff --git a/x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_field_column.tsx b/x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_field_column.tsx index 4e9611f7a8d2f..cdffe3742b534 100644 --- a/x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_field_column.tsx +++ b/x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_field_column.tsx @@ -4,14 +4,12 @@ * you may not use this file except in compliance with the Elastic License. */ -import stringify from 'json-stable-stringify'; -import React, { useMemo } from 'react'; - +import React from 'react'; import { euiStyled } from '../../../../../observability/public'; +import { LogColumn } from '../../../../common/http_api'; import { isFieldColumn, isHighlightFieldColumn } from '../../../utils/log_entry'; -import { ActiveHighlightMarker, highlightFieldValue, HighlightMarker } from './highlighting'; +import { FieldValue } from './field_value'; import { LogEntryColumnContent } from './log_entry_column'; -import { LogColumn } from '../../../../common/http_api'; import { longWrappedContentStyle, preWrappedContentStyle, @@ -32,44 +30,20 @@ export const LogEntryFieldColumn: React.FunctionComponent { - const value = useMemo(() => { - if (isFieldColumn(columnValue)) { - return columnValue.value; - } + if (isFieldColumn(columnValue)) { + return ( + + + + ); + } else { return null; - }, [columnValue]); - const formattedValue = Array.isArray(value) ? ( -
      - {value.map((entry, i) => ( - - {highlightFieldValue( - entry, - isHighlightFieldColumn(firstHighlight) ? firstHighlight.highlights : [], - isActiveHighlight ? ActiveHighlightMarker : HighlightMarker - )} - - ))} -
    - ) : ( - highlightFieldValue( - typeof value === 'string' ? value : stringify(value), - isHighlightFieldColumn(firstHighlight) ? firstHighlight.highlights : [], - isActiveHighlight ? ActiveHighlightMarker : HighlightMarker - ) - ); - - return {formattedValue}; -}; - -const CommaSeparatedLi = euiStyled.li` - display: inline; - &:not(:last-child) { - margin-right: 1ex; - &::after { - content: ','; - } } -`; +}; interface LogEntryColumnContentProps { wrapMode: WrapMode; diff --git a/x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_message_column.test.tsx b/x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_message_column.test.tsx new file mode 100644 index 0000000000000..b9871cc3b36f4 --- /dev/null +++ b/x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_message_column.test.tsx @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { render } from '@testing-library/react'; +import React from 'react'; +import { EuiThemeProvider } from '../../../../../observability/public'; +import { LogMessageColumn } from '../../../../common/http_api'; +import { LogEntryMessageColumn } from './log_entry_message_column'; + +describe('LogEntryMessageColumn', () => { + it('renders a single scalar field value without a wrapping list', () => { + const column: LogMessageColumn = { + columnId: 'TEST_COLUMN', + message: [{ field: 'TEST_FIELD', value: ['VALUE'], highlights: [] }], + }; + + const renderResult = render( + , + { wrapper: EuiThemeProvider } + ); + + expect(renderResult.getByTestId('LogEntryColumnContent')).toHaveTextContent(/^VALUE$/); + expect(renderResult.queryByTestId('LogEntryFieldValues')).toBe(null); + }); + + it('renders a single array of scalar field values as a list', () => { + const column: LogMessageColumn = { + columnId: 'TEST_COLUMN', + message: [{ field: 'TEST_FIELD', value: ['VALUE_1', 'VALUE_2'], highlights: [] }], + }; + + const renderResult = render( + , + { wrapper: EuiThemeProvider } + ); + + expect(renderResult.getByTestId('LogEntryFieldValues')).not.toBeEmptyDOMElement(); + expect(renderResult.getByTestId('LogEntryFieldValue-0')).toHaveTextContent('VALUE_1'); + expect(renderResult.getByTestId('LogEntryFieldValue-1')).toHaveTextContent('VALUE_2'); + }); + + it('renders a complex message with an array of complex field values', () => { + const column: LogMessageColumn = { + columnId: 'TEST_COLUMN', + message: [ + { constant: 'CONSTANT_1' }, + { field: 'TEST_FIELD', value: [{ lat: 1, lon: 2 }, 'VALUE_2'], highlights: [] }, + { constant: 'CONSTANT_2' }, + ], + }; + + const renderResult = render( + , + { wrapper: EuiThemeProvider } + ); + + expect(renderResult.getByTestId('LogEntryColumnContent')).toHaveTextContent( + /^CONSTANT_1.*{"lat":1,"lon":2}.*VALUE_2.*CONSTANT_2$/ + ); + expect(renderResult.getByTestId('LogEntryFieldValues')).not.toBeEmptyDOMElement(); + expect(renderResult.getByTestId('LogEntryFieldValue-0')).toHaveTextContent('{"lat":1,"lon":2}'); + expect(renderResult.getByTestId('LogEntryFieldValue-1')).toHaveTextContent('VALUE_2'); + }); +}); diff --git a/x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_message_column.tsx b/x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_message_column.tsx index f83a0a222d3dc..b7ff6b1409bd5 100644 --- a/x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_message_column.tsx +++ b/x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_message_column.tsx @@ -5,17 +5,16 @@ */ import React, { memo, useMemo } from 'react'; -import stringify from 'json-stable-stringify'; - import { euiStyled } from '../../../../../observability/public'; +import { LogColumn, LogMessagePart } from '../../../../common/http_api'; import { isConstantSegment, isFieldSegment, + isHighlightFieldSegment, isHighlightMessageColumn, isMessageColumn, - isHighlightFieldSegment, } from '../../../utils/log_entry'; -import { ActiveHighlightMarker, highlightFieldValue, HighlightMarker } from './highlighting'; +import { FieldValue } from './field_value'; import { LogEntryColumnContent } from './log_entry_column'; import { longWrappedContentStyle, @@ -23,7 +22,6 @@ import { unwrappedContentStyle, WrapMode, } from './text_styles'; -import { LogColumn, LogMessagePart } from '../../../../common/http_api'; interface LogEntryMessageColumnProps { columnValue: LogColumn; @@ -65,10 +63,10 @@ const formatMessageSegments = ( highlights: LogColumn[], isActiveHighlight: boolean ) => - messageSegments.map((messageSegment, index) => - formatMessageSegment( - messageSegment, - highlights.map((highlight) => { + messageSegments.map((messageSegment, index) => { + if (isFieldSegment(messageSegment)) { + // we only support one highlight for now + const [firstHighlight = []] = highlights.map((highlight) => { if (isHighlightMessageColumn(highlight)) { const segment = highlight.message[index]; if (isHighlightFieldSegment(segment)) { @@ -76,30 +74,19 @@ const formatMessageSegments = ( } } return []; - }), - isActiveHighlight - ) - ); + }); -const formatMessageSegment = ( - messageSegment: LogMessagePart, - [firstHighlight = []]: string[][], // we only support one highlight for now - isActiveHighlight: boolean -): React.ReactNode => { - if (isFieldSegment(messageSegment)) { - const value = - typeof messageSegment.value === 'string' - ? messageSegment.value - : stringify(messageSegment.value); - - return highlightFieldValue( - value, - firstHighlight, - isActiveHighlight ? ActiveHighlightMarker : HighlightMarker - ); - } else if (isConstantSegment(messageSegment)) { - return messageSegment.constant; - } + return ( + + ); + } else if (isConstantSegment(messageSegment)) { + return messageSegment.constant; + } - return 'failed to format message'; -}; + return 'failed to format message'; + }); diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/sections/top_categories/category_example_message.tsx b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/sections/top_categories/category_example_message.tsx index 7bcc05280994c..84d7e198636e9 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/sections/top_categories/category_example_message.tsx +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/sections/top_categories/category_example_message.tsx @@ -86,7 +86,7 @@ export const CategoryExampleMessage: React.FunctionComponent<{ = ({ = ({ columnValue={{ columnId: datasetColumnId, field: 'event.dataset', - value: humanFriendlyDataset, + value: [humanFriendlyDataset], highlights: [], }} highlights={noHighlights} diff --git a/x-pack/plugins/infra/server/lib/adapters/framework/adapter_types.ts b/x-pack/plugins/infra/server/lib/adapters/framework/adapter_types.ts index 0b7ee6e66db2f..f6b18a449c82a 100644 --- a/x-pack/plugins/infra/server/lib/adapters/framework/adapter_types.ts +++ b/x-pack/plugins/infra/server/lib/adapters/framework/adapter_types.ts @@ -15,6 +15,7 @@ import { PluginSetupContract as FeaturesPluginSetup } from '../../../../../../pl import { SpacesPluginSetup } from '../../../../../../plugins/spaces/server'; import { PluginSetupContract as AlertingPluginContract } from '../../../../../alerts/server'; import { MlPluginSetup } from '../../../../../ml/server'; +import { JsonArray, JsonValue } from '../../../../common/typed_json'; export interface InfraServerPluginDeps { home: HomeServerPluginSetup; @@ -108,7 +109,10 @@ export type SearchHit = SearchResponse['hits']['hits'][0]; export interface SortedSearchHit extends SearchHit { sort: any[]; _source: { - [field: string]: any; + [field: string]: JsonValue; + }; + fields: { + [field: string]: JsonArray; }; } diff --git a/x-pack/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts b/x-pack/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts index c5b667fb20538..9309ad85a3570 100644 --- a/x-pack/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts +++ b/x-pack/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts @@ -4,18 +4,17 @@ * you may not use this file except in compliance with the Elastic License. */ -/* eslint-disable @typescript-eslint/no-empty-interface */ - import { timeMilliseconds } from 'd3-time'; +import { fold, map } from 'fp-ts/lib/Either'; +import { constant, identity } from 'fp-ts/lib/function'; +import { pipe } from 'fp-ts/lib/pipeable'; import * as runtimeTypes from 'io-ts'; import { compact, first } from 'lodash'; -import { pipe } from 'fp-ts/lib/pipeable'; -import { map, fold } from 'fp-ts/lib/Either'; -import { identity, constant } from 'fp-ts/lib/function'; import { RequestHandlerContext } from 'src/core/server'; -import { JsonValue } from '../../../../common/typed_json'; +import { JsonArray } from '../../../../common/typed_json'; import { LogEntriesAdapter, + LogItemHit, LogEntriesParams, LogEntryDocument, LogEntryQuery, @@ -28,13 +27,6 @@ import { KibanaFramework } from '../framework/kibana_framework_adapter'; const TIMESTAMP_FORMAT = 'epoch_millis'; -interface LogItemHit { - _index: string; - _id: string; - fields: { [key: string]: [value: unknown] }; - sort: [number, number]; -} - export class InfraKibanaLogEntriesAdapter implements LogEntriesAdapter { constructor(private readonly framework: KibanaFramework) {} @@ -231,13 +223,14 @@ export class InfraKibanaLogEntriesAdapter implements LogEntriesAdapter { function mapHitsToLogEntryDocuments(hits: SortedSearchHit[], fields: string[]): LogEntryDocument[] { return hits.map((hit) => { - const logFields = fields.reduce<{ [fieldName: string]: JsonValue }>( - (flattenedFields, field) => { - if (field in hit.fields) { - flattenedFields[field] = hit.fields[field][0]; - } - return flattenedFields; - }, + const logFields = fields.reduce<{ [fieldName: string]: JsonArray }>( + (flattenedFields, field) => + field in hit.fields + ? { + ...flattenedFields, + [field]: hit.fields[field], + } + : flattenedFields, {} ); @@ -338,8 +331,9 @@ const LogSummaryDateRangeBucketRuntimeType = runtimeTypes.intersection([ }), ]); -export interface LogSummaryDateRangeBucket - extends runtimeTypes.TypeOf {} +export type LogSummaryDateRangeBucket = runtimeTypes.TypeOf< + typeof LogSummaryDateRangeBucketRuntimeType +>; const LogSummaryResponseRuntimeType = runtimeTypes.type({ aggregations: runtimeTypes.type({ @@ -349,5 +343,4 @@ const LogSummaryResponseRuntimeType = runtimeTypes.type({ }), }); -export interface LogSummaryResponse - extends runtimeTypes.TypeOf {} +export type LogSummaryResponse = runtimeTypes.TypeOf; diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_apache2.test.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_apache2.test.ts index 367ae6a0cae89..e7e381dc7f8f6 100644 --- a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_apache2.test.ts +++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_apache2.test.ts @@ -13,251 +13,290 @@ describe('Filebeat Rules', () => { describe('in ECS format', () => { test('Apache2 Access', () => { const flattenedDocument = { - '@timestamp': '2016-12-26T16:22:13.000Z', - 'ecs.version': '1.0.0-beta2', - 'event.dataset': 'apache.access', - 'event.module': 'apache', - 'fileset.name': 'access', - 'http.request.method': 'GET', - 'http.request.referrer': '-', - 'http.response.body.bytes': 499, - 'http.response.status_code': 404, - 'http.version': '1.1', - 'input.type': 'log', - 'log.offset': 73, - 'service.type': 'apache', - 'source.address': '192.168.33.1', - 'source.ip': '192.168.33.1', - 'url.original': '/hello', - 'user.name': '-', - 'user_agent.device': 'Other', - 'user_agent.major': '50', - 'user_agent.minor': '0', - 'user_agent.name': 'Firefox', - 'user_agent.original': + '@timestamp': ['2016-12-26T16:22:13.000Z'], + 'ecs.version': ['1.0.0-beta2'], + 'event.dataset': ['apache.access'], + 'event.module': ['apache'], + 'fileset.name': ['access'], + 'http.request.method': ['GET'], + 'http.request.referrer': ['-'], + 'http.response.body.bytes': [499], + 'http.response.status_code': [404], + 'http.version': ['1.1'], + 'input.type': ['log'], + 'log.offset': [73], + 'service.type': ['apache'], + 'source.address': ['192.168.33.1'], + 'source.ip': ['192.168.33.1'], + 'url.original': ['/hello'], + 'user.name': ['-'], + 'user_agent.device': ['Other'], + 'user_agent.major': ['50'], + 'user_agent.minor': ['0'], + 'user_agent.name': ['Firefox'], + 'user_agent.original': [ 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0', - 'user_agent.os.full_name': 'Mac OS X 10.12', - 'user_agent.os.major': '10', - 'user_agent.os.minor': '12', - 'user_agent.os.name': 'Mac OS X', + ], + 'user_agent.os.full_name': ['Mac OS X 10.12'], + 'user_agent.os.major': ['10'], + 'user_agent.os.minor': ['12'], + 'user_agent.os.name': ['Mac OS X'], }; const highlights = { 'http.request.method': ['GET'], }; expect(format(flattenedDocument, highlights)).toMatchInlineSnapshot(` -Array [ - Object { - "constant": "[", - }, - Object { - "field": "event.module", - "highlights": Array [], - "value": "apache", - }, - Object { - "constant": "][access] ", - }, - Object { - "field": "source.ip", - "highlights": Array [], - "value": "192.168.33.1", - }, - Object { - "constant": " ", - }, - Object { - "field": "user.name", - "highlights": Array [], - "value": "-", - }, - Object { - "constant": " \\"", - }, - Object { - "field": "http.request.method", - "highlights": Array [ - "GET", - ], - "value": "GET", - }, - Object { - "constant": " ", - }, - Object { - "field": "url.original", - "highlights": Array [], - "value": "/hello", - }, - Object { - "constant": " HTTP/", - }, - Object { - "field": "http.version", - "highlights": Array [], - "value": "1.1", - }, - Object { - "constant": "\\" ", - }, - Object { - "field": "http.response.status_code", - "highlights": Array [], - "value": "404", - }, - Object { - "constant": " ", - }, - Object { - "field": "http.response.body.bytes", - "highlights": Array [], - "value": "499", - }, -] -`); + Array [ + Object { + "constant": "[", + }, + Object { + "field": "event.module", + "highlights": Array [], + "value": Array [ + "apache", + ], + }, + Object { + "constant": "][access] ", + }, + Object { + "field": "source.ip", + "highlights": Array [], + "value": Array [ + "192.168.33.1", + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "user.name", + "highlights": Array [], + "value": Array [ + "-", + ], + }, + Object { + "constant": " \\"", + }, + Object { + "field": "http.request.method", + "highlights": Array [ + "GET", + ], + "value": Array [ + "GET", + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "url.original", + "highlights": Array [], + "value": Array [ + "/hello", + ], + }, + Object { + "constant": " HTTP/", + }, + Object { + "field": "http.version", + "highlights": Array [], + "value": Array [ + "1.1", + ], + }, + Object { + "constant": "\\" ", + }, + Object { + "field": "http.response.status_code", + "highlights": Array [], + "value": Array [ + 404, + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "http.response.body.bytes", + "highlights": Array [], + "value": Array [ + 499, + ], + }, + ] + `); }); test('Apache2 Error', () => { const flattenedDocument = { - '@timestamp': '2016-12-26T16:22:08.000Z', - 'ecs.version': '1.0.0-beta2', - 'event.dataset': 'apache.error', - 'event.module': 'apache', - 'fileset.name': 'error', - 'input.type': 'log', - 'log.level': 'error', - 'log.offset': 0, - message: 'File does not exist: /var/www/favicon.ico', - 'service.type': 'apache', - 'source.address': '192.168.33.1', - 'source.ip': '192.168.33.1', + '@timestamp': ['2016-12-26T16:22:08.000Z'], + 'ecs.version': ['1.0.0-beta2'], + 'event.dataset': ['apache.error'], + 'event.module': ['apache'], + 'fileset.name': ['error'], + 'input.type': ['log'], + 'log.level': ['error'], + 'log.offset': [0], + message: ['File does not exist: /var/www/favicon.ico'], + 'service.type': ['apache'], + 'source.address': ['192.168.33.1'], + 'source.ip': ['192.168.33.1'], }; expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` -Array [ - Object { - "constant": "[apache][", - }, - Object { - "field": "log.level", - "highlights": Array [], - "value": "error", - }, - Object { - "constant": "] ", - }, - Object { - "field": "message", - "highlights": Array [], - "value": "File does not exist: /var/www/favicon.ico", - }, -] -`); + Array [ + Object { + "constant": "[apache][", + }, + Object { + "field": "log.level", + "highlights": Array [], + "value": Array [ + "error", + ], + }, + Object { + "constant": "] ", + }, + Object { + "field": "message", + "highlights": Array [], + "value": Array [ + "File does not exist: /var/www/favicon.ico", + ], + }, + ] + `); }); }); describe('in pre-ECS format', () => { test('Apache2 Access', () => { const flattenedDocument = { - 'apache2.access': true, - 'apache2.access.remote_ip': '192.168.1.42', - 'apache2.access.user_name': 'admin', - 'apache2.access.method': 'GET', - 'apache2.access.url': '/faqs', - 'apache2.access.http_version': '1.1', - 'apache2.access.response_code': '200', - 'apache2.access.body_sent.bytes': 1024, + 'apache2.access.remote_ip': ['192.168.1.42'], + 'apache2.access.user_name': ['admin'], + 'apache2.access.method': ['GET'], + 'apache2.access.url': ['/faqs'], + 'apache2.access.http_version': ['1.1'], + 'apache2.access.response_code': ['200'], + 'apache2.access.body_sent.bytes': [1024], }; expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` -Array [ - Object { - "constant": "[apache][access] ", - }, - Object { - "field": "apache2.access.remote_ip", - "highlights": Array [], - "value": "192.168.1.42", - }, - Object { - "constant": " ", - }, - Object { - "field": "apache2.access.user_name", - "highlights": Array [], - "value": "admin", - }, - Object { - "constant": " \\"", - }, - Object { - "field": "apache2.access.method", - "highlights": Array [], - "value": "GET", - }, - Object { - "constant": " ", - }, - Object { - "field": "apache2.access.url", - "highlights": Array [], - "value": "/faqs", - }, - Object { - "constant": " HTTP/", - }, - Object { - "field": "apache2.access.http_version", - "highlights": Array [], - "value": "1.1", - }, - Object { - "constant": "\\" ", - }, - Object { - "field": "apache2.access.response_code", - "highlights": Array [], - "value": "200", - }, - Object { - "constant": " ", - }, - Object { - "field": "apache2.access.body_sent.bytes", - "highlights": Array [], - "value": "1024", - }, -] -`); + Array [ + Object { + "constant": "[apache][access] ", + }, + Object { + "field": "apache2.access.remote_ip", + "highlights": Array [], + "value": Array [ + "192.168.1.42", + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "apache2.access.user_name", + "highlights": Array [], + "value": Array [ + "admin", + ], + }, + Object { + "constant": " \\"", + }, + Object { + "field": "apache2.access.method", + "highlights": Array [], + "value": Array [ + "GET", + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "apache2.access.url", + "highlights": Array [], + "value": Array [ + "/faqs", + ], + }, + Object { + "constant": " HTTP/", + }, + Object { + "field": "apache2.access.http_version", + "highlights": Array [], + "value": Array [ + "1.1", + ], + }, + Object { + "constant": "\\" ", + }, + Object { + "field": "apache2.access.response_code", + "highlights": Array [], + "value": Array [ + "200", + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "apache2.access.body_sent.bytes", + "highlights": Array [], + "value": Array [ + 1024, + ], + }, + ] + `); }); test('Apache2 Error', () => { const flattenedDocument = { - 'apache2.error.message': + 'apache2.error.message': [ 'AH00489: Apache/2.4.18 (Ubuntu) configured -- resuming normal operations', - 'apache2.error.level': 'notice', + ], + 'apache2.error.level': ['notice'], }; expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` -Array [ - Object { - "constant": "[apache][", - }, - Object { - "field": "apache2.error.level", - "highlights": Array [], - "value": "notice", - }, - Object { - "constant": "] ", - }, - Object { - "field": "apache2.error.message", - "highlights": Array [], - "value": "AH00489: Apache/2.4.18 (Ubuntu) configured -- resuming normal operations", - }, -] -`); + Array [ + Object { + "constant": "[apache][", + }, + Object { + "field": "apache2.error.level", + "highlights": Array [], + "value": Array [ + "notice", + ], + }, + Object { + "constant": "] ", + }, + Object { + "field": "apache2.error.message", + "highlights": Array [], + "value": Array [ + "AH00489: Apache/2.4.18 (Ubuntu) configured -- resuming normal operations", + ], + }, + ] + `); }); }); }); diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_apache2.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_apache2.ts index fe7ebffe91329..3698753ef42ea 100644 --- a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_apache2.ts +++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_apache2.ts @@ -8,7 +8,7 @@ export const filebeatApache2Rules = [ { // pre-ECS when: { - exists: ['apache2.access'], + existsPrefix: ['apache2.access'], }, format: [ { diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_auditd.test.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_auditd.test.ts index aa490c595d9fd..4481ff434802f 100644 --- a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_auditd.test.ts +++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_auditd.test.ts @@ -13,345 +13,605 @@ describe('Filebeat Rules', () => { describe('in ECS format', () => { test('auditd log with outcome', () => { const flattenedDocument = { - '@timestamp': '2016-12-07T02:17:21.515Z', - 'auditd.log': { - addr: '96.241.146.97', - cipher: 'chacha20-poly1305@openssh.com', - direction: 'from-server', - ksize: '512', - laddr: '10.142.0.2', - lport: '22', - pfs: 'curve25519-sha256@libssh.org', - rport: '63927', - sequence: 406, - ses: '4294967295', - spid: '1299', - subj: 'system_u:system_r:sshd_t:s0-s0:c0.c1023', - }, - 'ecs.version': '1.0.0-beta2', - 'event.action': 'crypto_session', - 'event.dataset': 'auditd.log', - 'event.module': 'auditd', - 'event.outcome': 'success', - 'fileset.name': 'log', - 'input.type': 'log', - 'log.offset': 783, - message: 'op=start', - process: { executable: '/usr/sbin/sshd', pid: 1298 }, - 'service.type': 'auditd', - user: { 'audit.id': '4294967295', id: '0', 'saved.id': '74' }, + '@timestamp': ['2016-12-07T02:17:21.515Z'], + 'auditd.log.addr': ['96.241.146.97'], + 'auditd.log.cipher': ['chacha20-poly1305@openssh.com'], + 'auditd.log.direction': ['from-server'], + 'auditd.log.ksize': ['512'], + 'auditd.log.laddr': ['10.142.0.2'], + 'auditd.log.lport': ['22'], + 'auditd.log.pfs': ['curve25519-sha256@libssh.org'], + 'auditd.log.rport': ['63927'], + 'auditd.log.sequence': [406], + 'auditd.log.ses': ['4294967295'], + 'auditd.log.spid': ['1299'], + 'auditd.log.subj': ['system_u:system_r:sshd_t:s0-s0:c0.c1023'], + 'ecs.version': ['1.0.0-beta2'], + 'event.action': ['crypto_session'], + 'event.dataset': ['auditd.log'], + 'event.module': ['auditd'], + 'event.outcome': ['success'], + 'fileset.name': ['log'], + 'input.type': ['log'], + 'log.offset': [783], + message: ['op=start'], + 'process.executable': ['/usr/sbin/sshd'], + 'process.pid': [1298], + 'service.type': ['auditd'], + 'user.audit.id': ['4294967295'], + 'user.id': ['0'], + 'user.saved.id': ['74'], }; expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` -Array [ - Object { - "constant": "[AuditD][", - }, - Object { - "field": "event.action", - "highlights": Array [], - "value": "crypto_session", - }, - Object { - "constant": "]", - }, - Object { - "constant": " ", - }, - Object { - "field": "event.outcome", - "highlights": Array [], - "value": "success", - }, - Object { - "constant": " ", - }, - Object { - "constant": "user", - }, - Object { - "constant": "=", - }, - Object { - "field": "user", - "highlights": Array [], - "value": "{\\"audit.id\\":\\"4294967295\\",\\"id\\":\\"0\\",\\"saved.id\\":\\"74\\"}", - }, - Object { - "constant": " ", - }, - Object { - "constant": "process", - }, - Object { - "constant": "=", - }, - Object { - "field": "process", - "highlights": Array [], - "value": "{\\"executable\\":\\"/usr/sbin/sshd\\",\\"pid\\":1298}", - }, - Object { - "constant": " ", - }, - Object { - "field": "auditd.log", - "highlights": Array [], - "value": "{\\"addr\\":\\"96.241.146.97\\",\\"cipher\\":\\"chacha20-poly1305@openssh.com\\",\\"direction\\":\\"from-server\\",\\"ksize\\":\\"512\\",\\"laddr\\":\\"10.142.0.2\\",\\"lport\\":\\"22\\",\\"pfs\\":\\"curve25519-sha256@libssh.org\\",\\"rport\\":\\"63927\\",\\"sequence\\":406,\\"ses\\":\\"4294967295\\",\\"spid\\":\\"1299\\",\\"subj\\":\\"system_u:system_r:sshd_t:s0-s0:c0.c1023\\"}", - }, - Object { - "constant": " ", - }, - Object { - "field": "message", - "highlights": Array [], - "value": "op=start", - }, -] -`); + Array [ + Object { + "constant": "[AuditD][", + }, + Object { + "field": "event.action", + "highlights": Array [], + "value": Array [ + "crypto_session", + ], + }, + Object { + "constant": "]", + }, + Object { + "constant": " ", + }, + Object { + "field": "event.outcome", + "highlights": Array [], + "value": Array [ + "success", + ], + }, + Object { + "constant": " ", + }, + Object { + "constant": "user", + }, + Object { + "constant": "=", + }, + Object { + "field": "user.audit.id", + "highlights": Array [], + "value": Array [ + "4294967295", + ], + }, + Object { + "field": "user.id", + "highlights": Array [], + "value": Array [ + "0", + ], + }, + Object { + "field": "user.saved.id", + "highlights": Array [], + "value": Array [ + "74", + ], + }, + Object { + "constant": " ", + }, + Object { + "constant": "process", + }, + Object { + "constant": "=", + }, + Object { + "field": "process.executable", + "highlights": Array [], + "value": Array [ + "/usr/sbin/sshd", + ], + }, + Object { + "field": "process.pid", + "highlights": Array [], + "value": Array [ + 1298, + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "auditd.log.addr", + "highlights": Array [], + "value": Array [ + "96.241.146.97", + ], + }, + Object { + "field": "auditd.log.cipher", + "highlights": Array [], + "value": Array [ + "chacha20-poly1305@openssh.com", + ], + }, + Object { + "field": "auditd.log.direction", + "highlights": Array [], + "value": Array [ + "from-server", + ], + }, + Object { + "field": "auditd.log.ksize", + "highlights": Array [], + "value": Array [ + "512", + ], + }, + Object { + "field": "auditd.log.laddr", + "highlights": Array [], + "value": Array [ + "10.142.0.2", + ], + }, + Object { + "field": "auditd.log.lport", + "highlights": Array [], + "value": Array [ + "22", + ], + }, + Object { + "field": "auditd.log.pfs", + "highlights": Array [], + "value": Array [ + "curve25519-sha256@libssh.org", + ], + }, + Object { + "field": "auditd.log.rport", + "highlights": Array [], + "value": Array [ + "63927", + ], + }, + Object { + "field": "auditd.log.sequence", + "highlights": Array [], + "value": Array [ + 406, + ], + }, + Object { + "field": "auditd.log.ses", + "highlights": Array [], + "value": Array [ + "4294967295", + ], + }, + Object { + "field": "auditd.log.spid", + "highlights": Array [], + "value": Array [ + "1299", + ], + }, + Object { + "field": "auditd.log.subj", + "highlights": Array [], + "value": Array [ + "system_u:system_r:sshd_t:s0-s0:c0.c1023", + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "message", + "highlights": Array [], + "value": Array [ + "op=start", + ], + }, + ] + `); }); test('auditd log without outcome', () => { const flattenedDocument = { - '@timestamp': '2017-01-31T20:17:14.891Z', - 'auditd.log': { - a0: '9', - a1: '7f564b2672a0', - a2: 'b8', - a3: '0', - exit: '184', - items: '0', - sequence: 18877199, - ses: '4294967295', - success: 'yes', - syscall: '44', - tty: '(none)', - }, - 'ecs.version': '1.0.0-beta2', - 'event.action': 'syscall', - 'event.dataset': 'auditd.log', - 'event.module': 'auditd', - 'fileset.name': 'log', - 'host.architecture': 'x86_64', - 'input.type': 'log', - 'log.offset': 174, - process: { - executable: '/usr/libexec/strongswan/charon (deleted)', - name: 'charon', - pid: 1281, - ppid: 1240, - }, - 'service.type': 'auditd', - user: { - 'audit.id': '4294967295', - 'effective.group.id': '0', - 'effective.id': '0', - 'filesystem.group.id': '0', - 'filesystem.id': '0', - 'group.id': '0', - id: '0', - 'saved.group.id': '0', - 'saved.id': '0', - }, + '@timestamp': ['2017-01-31T20:17:14.891Z'], + 'auditd.log.a0': ['9'], + 'auditd.log.a1': ['7f564b2672a0'], + 'auditd.log.a2': ['b8'], + 'auditd.log.a3': ['0'], + 'auditd.log.exit': ['184'], + 'auditd.log.items': ['0'], + 'auditd.log.sequence': [18877199], + 'auditd.log.ses': ['4294967295'], + 'auditd.log.success': ['yes'], + 'auditd.log.syscall': ['44'], + 'auditd.log.tty': ['(none)'], + 'ecs.version': ['1.0.0-beta2'], + 'event.action': ['syscall'], + 'event.dataset': ['auditd.log'], + 'event.module': ['auditd'], + 'fileset.name': ['log'], + 'host.architecture': ['x86_64'], + 'input.type': ['log'], + 'log.offset': [174], + 'process.executable': ['/usr/libexec/strongswan/charon (deleted)'], + 'process.name': ['charon'], + 'process.pid': [1281], + 'process.ppid': [1240], + 'service.type': ['auditd'], + 'user.audit.id': ['4294967295'], + 'user.effective.group.id': ['0'], + 'user.effective.id': ['0'], + 'user.filesystem.group.id': ['0'], + 'user.filesystem.id': ['0'], + 'user.group.id': ['0'], + 'user.id': ['0'], + 'user.saved.group.id': ['0'], + 'user.saved.id': ['0'], }; expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` -Array [ - Object { - "constant": "[AuditD][", - }, - Object { - "field": "event.action", - "highlights": Array [], - "value": "syscall", - }, - Object { - "constant": "]", - }, - Object { - "constant": " ", - }, - Object { - "constant": "user", - }, - Object { - "constant": "=", - }, - Object { - "field": "user", - "highlights": Array [], - "value": "{\\"audit.id\\":\\"4294967295\\",\\"effective.group.id\\":\\"0\\",\\"effective.id\\":\\"0\\",\\"filesystem.group.id\\":\\"0\\",\\"filesystem.id\\":\\"0\\",\\"group.id\\":\\"0\\",\\"id\\":\\"0\\",\\"saved.group.id\\":\\"0\\",\\"saved.id\\":\\"0\\"}", - }, - Object { - "constant": " ", - }, - Object { - "constant": "process", - }, - Object { - "constant": "=", - }, - Object { - "field": "process", - "highlights": Array [], - "value": "{\\"executable\\":\\"/usr/libexec/strongswan/charon (deleted)\\",\\"name\\":\\"charon\\",\\"pid\\":1281,\\"ppid\\":1240}", - }, - Object { - "constant": " ", - }, - Object { - "field": "auditd.log", - "highlights": Array [], - "value": "{\\"a0\\":\\"9\\",\\"a1\\":\\"7f564b2672a0\\",\\"a2\\":\\"b8\\",\\"a3\\":\\"0\\",\\"exit\\":\\"184\\",\\"items\\":\\"0\\",\\"sequence\\":18877199,\\"ses\\":\\"4294967295\\",\\"success\\":\\"yes\\",\\"syscall\\":\\"44\\",\\"tty\\":\\"(none)\\"}", - }, - Object { - "constant": " ", - }, - Object { - "field": "message", - "highlights": Array [], - "value": "undefined", - }, -] -`); + Array [ + Object { + "constant": "[AuditD][", + }, + Object { + "field": "event.action", + "highlights": Array [], + "value": Array [ + "syscall", + ], + }, + Object { + "constant": "]", + }, + Object { + "constant": " ", + }, + Object { + "constant": "user", + }, + Object { + "constant": "=", + }, + Object { + "field": "user.audit.id", + "highlights": Array [], + "value": Array [ + "4294967295", + ], + }, + Object { + "field": "user.effective.group.id", + "highlights": Array [], + "value": Array [ + "0", + ], + }, + Object { + "field": "user.effective.id", + "highlights": Array [], + "value": Array [ + "0", + ], + }, + Object { + "field": "user.filesystem.group.id", + "highlights": Array [], + "value": Array [ + "0", + ], + }, + Object { + "field": "user.filesystem.id", + "highlights": Array [], + "value": Array [ + "0", + ], + }, + Object { + "field": "user.group.id", + "highlights": Array [], + "value": Array [ + "0", + ], + }, + Object { + "field": "user.id", + "highlights": Array [], + "value": Array [ + "0", + ], + }, + Object { + "field": "user.saved.group.id", + "highlights": Array [], + "value": Array [ + "0", + ], + }, + Object { + "field": "user.saved.id", + "highlights": Array [], + "value": Array [ + "0", + ], + }, + Object { + "constant": " ", + }, + Object { + "constant": "process", + }, + Object { + "constant": "=", + }, + Object { + "field": "process.executable", + "highlights": Array [], + "value": Array [ + "/usr/libexec/strongswan/charon (deleted)", + ], + }, + Object { + "field": "process.name", + "highlights": Array [], + "value": Array [ + "charon", + ], + }, + Object { + "field": "process.pid", + "highlights": Array [], + "value": Array [ + 1281, + ], + }, + Object { + "field": "process.ppid", + "highlights": Array [], + "value": Array [ + 1240, + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "auditd.log.a0", + "highlights": Array [], + "value": Array [ + "9", + ], + }, + Object { + "field": "auditd.log.a1", + "highlights": Array [], + "value": Array [ + "7f564b2672a0", + ], + }, + Object { + "field": "auditd.log.a2", + "highlights": Array [], + "value": Array [ + "b8", + ], + }, + Object { + "field": "auditd.log.a3", + "highlights": Array [], + "value": Array [ + "0", + ], + }, + Object { + "field": "auditd.log.exit", + "highlights": Array [], + "value": Array [ + "184", + ], + }, + Object { + "field": "auditd.log.items", + "highlights": Array [], + "value": Array [ + "0", + ], + }, + Object { + "field": "auditd.log.sequence", + "highlights": Array [], + "value": Array [ + 18877199, + ], + }, + Object { + "field": "auditd.log.ses", + "highlights": Array [], + "value": Array [ + "4294967295", + ], + }, + Object { + "field": "auditd.log.success", + "highlights": Array [], + "value": Array [ + "yes", + ], + }, + Object { + "field": "auditd.log.syscall", + "highlights": Array [], + "value": Array [ + "44", + ], + }, + Object { + "field": "auditd.log.tty", + "highlights": Array [], + "value": Array [ + "(none)", + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "message", + "highlights": Array [], + "value": Array [], + }, + ] + `); }); }); describe('in pre-ECS format', () => { test('auditd IPSEC rule', () => { const event = { - '@timestamp': '2017-01-31T20:17:14.891Z', - 'auditd.log.auid': '4294967295', - 'auditd.log.dst': '192.168.0.0', - 'auditd.log.dst_prefixlen': '16', - 'auditd.log.op': 'SPD-delete', - 'auditd.log.record_type': 'MAC_IPSEC_EVENT', - 'auditd.log.res': '1', - 'auditd.log.sequence': 18877201, - 'auditd.log.ses': '4294967295', - 'auditd.log.src': '192.168.2.0', - 'auditd.log.src_prefixlen': '24', - 'ecs.version': '1.0.0-beta2', - 'event.dataset': 'auditd.log', - 'event.module': 'auditd', - 'fileset.name': 'log', - 'input.type': 'log', - 'log.offset': 0, + '@timestamp': ['2017-01-31T20:17:14.891Z'], + 'auditd.log.auid': ['4294967295'], + 'auditd.log.dst': ['192.168.0.0'], + 'auditd.log.dst_prefixlen': ['16'], + 'auditd.log.op': ['SPD-delete'], + 'auditd.log.record_type': ['MAC_IPSEC_EVENT'], + 'auditd.log.res': ['1'], + 'auditd.log.sequence': [18877201], + 'auditd.log.ses': ['4294967295'], + 'auditd.log.src': ['192.168.2.0'], + 'auditd.log.src_prefixlen': ['24'], + 'ecs.version': ['1.0.0-beta2'], + 'event.dataset': ['auditd.log'], + 'event.module': ['auditd'], + 'fileset.name': ['log'], + 'input.type': ['log'], + 'log.offset': [0], }; const message = format(event, {}); expect(message).toEqual([ { constant: '[AuditD][' }, - { field: 'auditd.log.record_type', highlights: [], value: 'MAC_IPSEC_EVENT' }, + { field: 'auditd.log.record_type', highlights: [], value: ['MAC_IPSEC_EVENT'] }, { constant: '] src:' }, - { field: 'auditd.log.src', highlights: [], value: '192.168.2.0' }, + { field: 'auditd.log.src', highlights: [], value: ['192.168.2.0'] }, { constant: ' dst:' }, - { field: 'auditd.log.dst', highlights: [], value: '192.168.0.0' }, + { field: 'auditd.log.dst', highlights: [], value: ['192.168.0.0'] }, { constant: ' op:' }, - { field: 'auditd.log.op', highlights: [], value: 'SPD-delete' }, + { field: 'auditd.log.op', highlights: [], value: ['SPD-delete'] }, ]); }); test('AuditD SYSCALL rule', () => { const event = { - '@timestamp': '2017-01-31T20:17:14.891Z', - 'auditd.log.a0': '9', - 'auditd.log.a1': '7f564b2672a0', - 'auditd.log.a2': 'b8', - 'auditd.log.a3': '0', - 'auditd.log.arch': 'x86_64', - 'auditd.log.auid': '4294967295', - 'auditd.log.comm': 'charon', - 'auditd.log.egid': '0', - 'auditd.log.euid': '0', - 'auditd.log.exe': '/usr/libexec/strongswan/charon (deleted)', - 'auditd.log.exit': '184', - 'auditd.log.fsgid': '0', - 'auditd.log.fsuid': '0', - 'auditd.log.gid': '0', - 'auditd.log.items': '0', - 'auditd.log.pid': '1281', - 'auditd.log.ppid': '1240', - 'auditd.log.record_type': 'SYSCALL', - 'auditd.log.sequence': 18877199, - 'auditd.log.ses': '4294967295', - 'auditd.log.sgid': '0', - 'auditd.log.success': 'yes', - 'auditd.log.suid': '0', - 'auditd.log.syscall': '44', - 'auditd.log.tty': '(none)', - 'auditd.log.uid': '0', - 'ecs.version': '1.0.0-beta2', - 'event.dataset': 'auditd.log', - 'event.module': 'auditd', - 'fileset.name': 'log', - 'input.type': 'log', - 'log.offset': 174, + '@timestamp': ['2017-01-31T20:17:14.891Z'], + 'auditd.log.a0': ['9'], + 'auditd.log.a1': ['7f564b2672a0'], + 'auditd.log.a2': ['b8'], + 'auditd.log.a3': ['0'], + 'auditd.log.arch': ['x86_64'], + 'auditd.log.auid': ['4294967295'], + 'auditd.log.comm': ['charon'], + 'auditd.log.egid': ['0'], + 'auditd.log.euid': ['0'], + 'auditd.log.exe': ['/usr/libexec/strongswan/charon (deleted)'], + 'auditd.log.exit': ['184'], + 'auditd.log.fsgid': ['0'], + 'auditd.log.fsuid': ['0'], + 'auditd.log.gid': ['0'], + 'auditd.log.items': ['0'], + 'auditd.log.pid': ['1281'], + 'auditd.log.ppid': ['1240'], + 'auditd.log.record_type': ['SYSCALL'], + 'auditd.log.sequence': [18877199], + 'auditd.log.ses': ['4294967295'], + 'auditd.log.sgid': ['0'], + 'auditd.log.success': ['yes'], + 'auditd.log.suid': ['0'], + 'auditd.log.syscall': ['44'], + 'auditd.log.tty': ['(none)'], + 'auditd.log.uid': ['0'], + 'ecs.version': ['1.0.0-beta2'], + 'event.dataset': ['auditd.log'], + 'event.module': ['auditd'], + 'fileset.name': ['log'], + 'input.type': ['log'], + 'log.offset': [174], }; const message = format(event, {}); expect(message).toEqual([ { constant: '[AuditD][' }, - { field: 'auditd.log.record_type', highlights: [], value: 'SYSCALL' }, + { field: 'auditd.log.record_type', highlights: [], value: ['SYSCALL'] }, { constant: '] exe:' }, { field: 'auditd.log.exe', highlights: [], - value: '/usr/libexec/strongswan/charon (deleted)', + value: ['/usr/libexec/strongswan/charon (deleted)'], }, { constant: ' gid:' }, - { field: 'auditd.log.gid', highlights: [], value: '0' }, + { field: 'auditd.log.gid', highlights: [], value: ['0'] }, { constant: ' uid:' }, - { field: 'auditd.log.uid', highlights: [], value: '0' }, + { field: 'auditd.log.uid', highlights: [], value: ['0'] }, { constant: ' tty:' }, - { field: 'auditd.log.tty', highlights: [], value: '(none)' }, + { field: 'auditd.log.tty', highlights: [], value: ['(none)'] }, { constant: ' pid:' }, - { field: 'auditd.log.pid', highlights: [], value: '1281' }, + { field: 'auditd.log.pid', highlights: [], value: ['1281'] }, { constant: ' ppid:' }, - { field: 'auditd.log.ppid', highlights: [], value: '1240' }, + { field: 'auditd.log.ppid', highlights: [], value: ['1240'] }, ]); }); test('AuditD events with msg rule', () => { const event = { - '@timestamp': '2017-01-31T20:17:14.891Z', - 'auditd.log.auid': '4294967295', - 'auditd.log.record_type': 'EXAMPLE', - 'auditd.log.msg': 'some kind of message', - 'ecs.version': '1.0.0-beta2', - 'event.dataset': 'auditd.log', - 'event.module': 'auditd', - 'fileset.name': 'log', - 'input.type': 'log', - 'log.offset': 174, + '@timestamp': ['2017-01-31T20:17:14.891Z'], + 'auditd.log.auid': ['4294967295'], + 'auditd.log.record_type': ['EXAMPLE'], + 'auditd.log.msg': ['some kind of message'], + 'ecs.version': ['1.0.0-beta2'], + 'event.dataset': ['auditd.log'], + 'event.module': ['auditd'], + 'fileset.name': ['log'], + 'input.type': ['log'], + 'log.offset': [174], }; const message = format(event, {}); expect(message).toEqual([ { constant: '[AuditD][' }, - { field: 'auditd.log.record_type', highlights: [], value: 'EXAMPLE' }, + { field: 'auditd.log.record_type', highlights: [], value: ['EXAMPLE'] }, { constant: '] ' }, { field: 'auditd.log.msg', highlights: [], - value: 'some kind of message', + value: ['some kind of message'], }, ]); }); test('AuditD catchall rule', () => { const event = { - '@timestamp': '2017-01-31T20:17:14.891Z', - 'auditd.log.auid': '4294967295', - 'auditd.log.record_type': 'EXAMPLE', - 'ecs.version': '1.0.0-beta2', - 'event.dataset': 'auditd.log', - 'event.module': 'auditd', - 'fileset.name': 'log', - 'input.type': 'log', - 'log.offset': 174, + '@timestamp': ['2017-01-31T20:17:14.891Z'], + 'auditd.log.auid': ['4294967295'], + 'auditd.log.record_type': ['EXAMPLE'], + 'ecs.version': ['1.0.0-beta2'], + 'event.dataset': ['auditd.log'], + 'event.module': ['auditd'], + 'fileset.name': ['log'], + 'input.type': ['log'], + 'log.offset': [174], }; const message = format(event, {}); expect(message).toEqual([ { constant: '[AuditD][' }, - { field: 'auditd.log.record_type', highlights: [], value: 'EXAMPLE' }, + { field: 'auditd.log.record_type', highlights: [], value: ['EXAMPLE'] }, { constant: '] Event without message.' }, ]); }); diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_auditd.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_auditd.ts index d2557cf1599ce..cd64697db6d1c 100644 --- a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_auditd.ts +++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_auditd.ts @@ -4,24 +4,28 @@ * you may not use this file except in compliance with the Elastic License. */ -import { labelField } from './helpers'; +import { LogMessageFormattingRule } from '../rule_types'; +import { labelFieldsPrefix } from './helpers'; const commonActionField = [{ constant: '[AuditD][' }, { field: 'event.action' }, { constant: ']' }]; const commonOutcomeField = [{ constant: ' ' }, { field: 'event.outcome' }]; -export const filebeatAuditdRules = [ +export const filebeatAuditdRules: LogMessageFormattingRule[] = [ { // ECS format with outcome when: { - exists: ['ecs.version', 'event.action', 'event.outcome', 'auditd.log'], + all: [ + { exists: ['ecs.version', 'event.action', 'event.outcome'] }, + { existsPrefix: ['auditd.log'] }, + ], }, format: [ ...commonActionField, ...commonOutcomeField, - ...labelField('user', 'user'), - ...labelField('process', 'process'), + ...labelFieldsPrefix('user', 'user'), + ...labelFieldsPrefix('process', 'process'), { constant: ' ' }, - { field: 'auditd.log' }, + { fieldsPrefix: 'auditd.log' }, { constant: ' ' }, { field: 'message' }, ], @@ -29,14 +33,14 @@ export const filebeatAuditdRules = [ { // ECS format without outcome when: { - exists: ['ecs.version', 'event.action', 'auditd.log'], + all: [{ exists: ['ecs.version', 'event.action'] }, { existsPrefix: ['auditd.log'] }], }, format: [ ...commonActionField, - ...labelField('user', 'user'), - ...labelField('process', 'process'), + ...labelFieldsPrefix('user', 'user'), + ...labelFieldsPrefix('process', 'process'), { constant: ' ' }, - { field: 'auditd.log' }, + { fieldsPrefix: 'auditd.log' }, { constant: ' ' }, { field: 'message' }, ], @@ -44,10 +48,10 @@ export const filebeatAuditdRules = [ { // pre-ECS IPSEC_EVENT Rule when: { - exists: ['auditd.log.record_type', 'auditd.log.src', 'auditd.log.dst', 'auditd.log.op'], - values: { - 'auditd.log.record_type': 'MAC_IPSEC_EVENT', - }, + all: [ + { exists: ['auditd.log.record_type', 'auditd.log.src', 'auditd.log.dst', 'auditd.log.op'] }, + { values: { 'auditd.log.record_type': 'MAC_IPSEC_EVENT' } }, + ], }, format: [ { constant: '[AuditD][' }, @@ -63,18 +67,20 @@ export const filebeatAuditdRules = [ { // pre-ECS SYSCALL Rule when: { - exists: [ - 'auditd.log.record_type', - 'auditd.log.exe', - 'auditd.log.gid', - 'auditd.log.uid', - 'auditd.log.tty', - 'auditd.log.pid', - 'auditd.log.ppid', + all: [ + { + exists: [ + 'auditd.log.record_type', + 'auditd.log.exe', + 'auditd.log.gid', + 'auditd.log.uid', + 'auditd.log.tty', + 'auditd.log.pid', + 'auditd.log.ppid', + ], + }, + { values: { 'auditd.log.record_type': 'SYSCALL' } }, ], - values: { - 'auditd.log.record_type': 'SYSCALL', - }, }, format: [ { constant: '[AuditD][' }, diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_haproxy.test.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_haproxy.test.ts index 752b61684887e..40e8ea0fad857 100644 --- a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_haproxy.test.ts +++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_haproxy.test.ts @@ -13,779 +13,917 @@ describe('Filebeat Rules', () => { describe('in ECS format', () => { test('haproxy default log', () => { const flattenedDocument = { - 'destination.ip': '1.2.3.4', - 'destination.port': 5000, - 'ecs.version': '1.0.0-beta2', - 'event.dataset': 'haproxy.log', - 'event.module': 'haproxy', - 'fileset.name': 'log', - 'haproxy.frontend_name': 'main', - 'haproxy.mode': 'HTTP', - 'haproxy.source': '1.2.3.4', - 'input.type': 'log', - 'log.offset': 0, - 'process.name': 'haproxy', - 'process.pid': 24551, - 'service.type': 'haproxy', - 'source.address': '1.2.3.4', - 'source.geo.continent_name': 'North America', - 'source.geo.country_iso_code': 'US', - 'source.geo.location.lat': 37.751, - 'source.geo.location.lon': -97.822, - 'source.ip': '1.2.3.4', - 'source.port': 40780, + 'destination.ip': ['1.2.3.4'], + 'destination.port': [5000], + 'ecs.version': ['1.0.0-beta2'], + 'event.dataset': ['haproxy.log'], + 'event.module': ['haproxy'], + 'fileset.name': ['log'], + 'haproxy.frontend_name': ['main'], + 'haproxy.mode': ['HTTP'], + 'haproxy.source': ['1.2.3.4'], + 'input.type': ['log'], + 'log.offset': [0], + 'process.name': ['haproxy'], + 'process.pid': [24551], + 'service.type': ['haproxy'], + 'source.address': ['1.2.3.4'], + 'source.geo.continent_name': ['North America'], + 'source.geo.country_iso_code': ['US'], + 'source.geo.location.lat': [37.751], + 'source.geo.location.lon': [-97.822], + 'source.ip': ['1.2.3.4'], + 'source.port': [40780], }; expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` -Array [ - Object { - "constant": "[HAProxy] ", - }, - Object { - "field": "source.address", - "highlights": Array [], - "value": "1.2.3.4", - }, - Object { - "constant": ":", - }, - Object { - "field": "source.port", - "highlights": Array [], - "value": "40780", - }, - Object { - "constant": " ", - }, - Object { - "field": "haproxy.frontend_name", - "highlights": Array [], - "value": "main", - }, -] -`); + Array [ + Object { + "constant": "[HAProxy] ", + }, + Object { + "field": "source.address", + "highlights": Array [], + "value": Array [ + "1.2.3.4", + ], + }, + Object { + "constant": ":", + }, + Object { + "field": "source.port", + "highlights": Array [], + "value": Array [ + 40780, + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "haproxy.frontend_name", + "highlights": Array [], + "value": Array [ + "main", + ], + }, + ] + `); }); test('haproxy tcp log', () => { const flattenedDocument = { - 'ecs.version': '1.0.0-beta2', - 'event.dataset': 'haproxy.log', - 'event.duration': 1000000, - 'event.module': 'haproxy', - 'fileset.name': 'log', - 'haproxy.backend_name': 'app', - 'haproxy.backend_queue': 0, - 'haproxy.bytes_read': 212, - 'haproxy.connection_wait_time_ms': -1, - 'haproxy.connections.active': 1, - 'haproxy.connections.backend': 0, - 'haproxy.connections.frontend': 1, - 'haproxy.connections.retries': 0, - 'haproxy.connections.server': 0, - 'haproxy.frontend_name': 'main', - 'haproxy.server_name': '', - 'haproxy.server_queue': 0, - 'haproxy.source': '127.0.0.1', - 'haproxy.termination_state': 'SC', - 'haproxy.total_waiting_time_ms': -1, - 'input.type': 'log', - 'log.offset': 0, - 'process.name': 'haproxy', - 'process.pid': 25457, - 'service.type': 'haproxy', - 'source.address': '127.0.0.1', - 'source.ip': '127.0.0.1', - 'source.port': 40962, + 'ecs.version': ['1.0.0-beta2'], + 'event.dataset': ['haproxy.log'], + 'event.duration': [1000000], + 'event.module': ['haproxy'], + 'fileset.name': ['log'], + 'haproxy.backend_name': ['app'], + 'haproxy.backend_queue': [0], + 'haproxy.bytes_read': [212], + 'haproxy.connection_wait_time_ms': [-1], + 'haproxy.connections.active': [1], + 'haproxy.connections.backend': [0], + 'haproxy.connections.frontend': [1], + 'haproxy.connections.retries': [0], + 'haproxy.connections.server': [0], + 'haproxy.frontend_name': ['main'], + 'haproxy.server_name': [''], + 'haproxy.server_queue': [0], + 'haproxy.source': ['127.0.0.1'], + 'haproxy.termination_state': ['SC'], + 'haproxy.total_waiting_time_ms': [-1], + 'input.type': ['log'], + 'log.offset': [0], + 'process.name': ['haproxy'], + 'process.pid': [25457], + 'service.type': ['haproxy'], + 'source.address': ['127.0.0.1'], + 'source.ip': ['127.0.0.1'], + 'source.port': [40962], }; expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` -Array [ - Object { - "constant": "[HAProxy][tcp] ", - }, - Object { - "field": "source.address", - "highlights": Array [], - "value": "127.0.0.1", - }, - Object { - "constant": ":", - }, - Object { - "field": "source.port", - "highlights": Array [], - "value": "40962", - }, - Object { - "constant": " ", - }, - Object { - "field": "haproxy.frontend_name", - "highlights": Array [], - "value": "main", - }, - Object { - "constant": " -> ", - }, - Object { - "field": "haproxy.backend_name", - "highlights": Array [], - "value": "app", - }, - Object { - "constant": "/", - }, - Object { - "field": "haproxy.server_name", - "highlights": Array [], - "value": "", - }, - Object { - "constant": " ", - }, - Object { - "field": "haproxy.connections.active", - "highlights": Array [], - "value": "1", - }, - Object { - "constant": "/", - }, - Object { - "field": "haproxy.connections.frontend", - "highlights": Array [], - "value": "1", - }, - Object { - "constant": "/", - }, - Object { - "field": "haproxy.connections.backend", - "highlights": Array [], - "value": "0", - }, - Object { - "constant": "/", - }, - Object { - "field": "haproxy.connections.server", - "highlights": Array [], - "value": "0", - }, - Object { - "constant": "/", - }, - Object { - "field": "haproxy.connections.retries", - "highlights": Array [], - "value": "0", - }, - Object { - "constant": " ", - }, - Object { - "field": "haproxy.server_queue", - "highlights": Array [], - "value": "0", - }, - Object { - "constant": "/", - }, - Object { - "field": "haproxy.backend_queue", - "highlights": Array [], - "value": "0", - }, -] -`); + Array [ + Object { + "constant": "[HAProxy][tcp] ", + }, + Object { + "field": "source.address", + "highlights": Array [], + "value": Array [ + "127.0.0.1", + ], + }, + Object { + "constant": ":", + }, + Object { + "field": "source.port", + "highlights": Array [], + "value": Array [ + 40962, + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "haproxy.frontend_name", + "highlights": Array [], + "value": Array [ + "main", + ], + }, + Object { + "constant": " -> ", + }, + Object { + "field": "haproxy.backend_name", + "highlights": Array [], + "value": Array [ + "app", + ], + }, + Object { + "constant": "/", + }, + Object { + "field": "haproxy.server_name", + "highlights": Array [], + "value": Array [ + "", + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "haproxy.connections.active", + "highlights": Array [], + "value": Array [ + 1, + ], + }, + Object { + "constant": "/", + }, + Object { + "field": "haproxy.connections.frontend", + "highlights": Array [], + "value": Array [ + 1, + ], + }, + Object { + "constant": "/", + }, + Object { + "field": "haproxy.connections.backend", + "highlights": Array [], + "value": Array [ + 0, + ], + }, + Object { + "constant": "/", + }, + Object { + "field": "haproxy.connections.server", + "highlights": Array [], + "value": Array [ + 0, + ], + }, + Object { + "constant": "/", + }, + Object { + "field": "haproxy.connections.retries", + "highlights": Array [], + "value": Array [ + 0, + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "haproxy.server_queue", + "highlights": Array [], + "value": Array [ + 0, + ], + }, + Object { + "constant": "/", + }, + Object { + "field": "haproxy.backend_queue", + "highlights": Array [], + "value": Array [ + 0, + ], + }, + ] + `); }); test('haproxy http log', () => { const flattenedDocument = { - 'ecs.version': '1.0.0-beta2', - 'event.dataset': 'haproxy.log', - 'event.duration': 2000000, - 'event.module': 'haproxy', - 'fileset.name': 'log', - 'haproxy.backend_name': 'docs_microservice', - 'haproxy.backend_queue': 0, - 'haproxy.bytes_read': 168, - 'haproxy.connection_wait_time_ms': 1, - 'haproxy.connections.active': 6, - 'haproxy.connections.backend': 0, - 'haproxy.connections.frontend': 6, - 'haproxy.connections.retries': 0, - 'haproxy.connections.server': 0, - 'haproxy.frontend_name': 'incoming~', - 'haproxy.http.request.captured_cookie': '-', + 'ecs.version': ['1.0.0-beta2'], + 'event.dataset': ['haproxy.log'], + 'event.duration': [2000000], + 'event.module': ['haproxy'], + 'fileset.name': ['log'], + 'haproxy.backend_name': ['docs_microservice'], + 'haproxy.backend_queue': [0], + 'haproxy.bytes_read': [168], + 'haproxy.connection_wait_time_ms': [1], + 'haproxy.connections.active': [6], + 'haproxy.connections.backend': [0], + 'haproxy.connections.frontend': [6], + 'haproxy.connections.retries': [0], + 'haproxy.connections.server': [0], + 'haproxy.frontend_name': ['incoming~'], + 'haproxy.http.request.captured_cookie': ['-'], 'haproxy.http.request.captured_headers': ['docs.example.internal'], - 'haproxy.http.request.raw_request_line': + 'haproxy.http.request.raw_request_line': [ 'GET /component---src-pages-index-js-4b15624544f97cf0bb8f.js HTTP/1.1', - 'haproxy.http.request.time_wait_ms': 0, - 'haproxy.http.request.time_wait_without_data_ms': 0, - 'haproxy.http.response.captured_cookie': '-', + ], + 'haproxy.http.request.time_wait_ms': [0], + 'haproxy.http.request.time_wait_without_data_ms': [0], + 'haproxy.http.response.captured_cookie': ['-'], 'haproxy.http.response.captured_headers': [], - 'haproxy.server_name': 'docs', - 'haproxy.server_queue': 0, - 'haproxy.termination_state': '----', - 'haproxy.total_waiting_time_ms': 0, - 'http.response.bytes': 168, - 'http.response.status_code': 304, - 'input.type': 'log', - 'log.offset': 0, - 'process.name': 'haproxy', - 'process.pid': 32450, - 'service.type': 'haproxy', - 'source.address': '1.2.3.4', - 'source.geo.continent_name': 'North America', - 'source.geo.country_iso_code': 'US', - 'source.geo.location.lat': 37.751, - 'source.geo.location.lon': -97.822, - 'source.ip': '1.2.3.4', - 'source.port': 38862, + 'haproxy.server_name': ['docs'], + 'haproxy.server_queue': [0], + 'haproxy.termination_state': ['----'], + 'haproxy.total_waiting_time_ms': [0], + 'http.response.bytes': [168], + 'http.response.status_code': [304], + 'input.type': ['log'], + 'log.offset': [0], + 'process.name': ['haproxy'], + 'process.pid': [32450], + 'service.type': ['haproxy'], + 'source.address': ['1.2.3.4'], + 'source.geo.continent_name': ['North America'], + 'source.geo.country_iso_code': ['US'], + 'source.geo.location.lat': [37.751], + 'source.geo.location.lon': [-97.822], + 'source.ip': ['1.2.3.4'], + 'source.port': [38862], }; expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` -Array [ - Object { - "constant": "[HAProxy][http] ", - }, - Object { - "field": "source.address", - "highlights": Array [], - "value": "1.2.3.4", - }, - Object { - "constant": ":", - }, - Object { - "field": "source.port", - "highlights": Array [], - "value": "38862", - }, - Object { - "constant": " ", - }, - Object { - "field": "haproxy.frontend_name", - "highlights": Array [], - "value": "incoming~", - }, - Object { - "constant": " -> ", - }, - Object { - "field": "haproxy.backend_name", - "highlights": Array [], - "value": "docs_microservice", - }, - Object { - "constant": "/", - }, - Object { - "field": "haproxy.server_name", - "highlights": Array [], - "value": "docs", - }, - Object { - "constant": " \\"", - }, - Object { - "field": "haproxy.http.request.raw_request_line", - "highlights": Array [], - "value": "GET /component---src-pages-index-js-4b15624544f97cf0bb8f.js HTTP/1.1", - }, - Object { - "constant": "\\" ", - }, - Object { - "field": "http.response.status_code", - "highlights": Array [], - "value": "304", - }, - Object { - "constant": " ", - }, - Object { - "field": "haproxy.http.request.time_wait_ms", - "highlights": Array [], - "value": "0", - }, - Object { - "constant": "/", - }, - Object { - "field": "event.duration", - "highlights": Array [], - "value": "2000000", - }, - Object { - "constant": "/", - }, - Object { - "field": "haproxy.connection_wait_time_ms", - "highlights": Array [], - "value": "1", - }, - Object { - "constant": "/", - }, - Object { - "field": "haproxy.http.request.time_wait_without_data_ms", - "highlights": Array [], - "value": "0", - }, - Object { - "constant": "/", - }, - Object { - "field": "event.duration", - "highlights": Array [], - "value": "2000000", - }, - Object { - "constant": " ", - }, - Object { - "field": "haproxy.connections.active", - "highlights": Array [], - "value": "6", - }, - Object { - "constant": "/", - }, - Object { - "field": "haproxy.connections.frontend", - "highlights": Array [], - "value": "6", - }, - Object { - "constant": "/", - }, - Object { - "field": "haproxy.connections.backend", - "highlights": Array [], - "value": "0", - }, - Object { - "constant": "/", - }, - Object { - "field": "haproxy.connections.server", - "highlights": Array [], - "value": "0", - }, - Object { - "constant": "/", - }, - Object { - "field": "haproxy.connections.retries", - "highlights": Array [], - "value": "0", - }, - Object { - "constant": " ", - }, - Object { - "field": "haproxy.server_queue", - "highlights": Array [], - "value": "0", - }, - Object { - "constant": "/", - }, - Object { - "field": "haproxy.backend_queue", - "highlights": Array [], - "value": "0", - }, -] -`); + Array [ + Object { + "constant": "[HAProxy][http] ", + }, + Object { + "field": "source.address", + "highlights": Array [], + "value": Array [ + "1.2.3.4", + ], + }, + Object { + "constant": ":", + }, + Object { + "field": "source.port", + "highlights": Array [], + "value": Array [ + 38862, + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "haproxy.frontend_name", + "highlights": Array [], + "value": Array [ + "incoming~", + ], + }, + Object { + "constant": " -> ", + }, + Object { + "field": "haproxy.backend_name", + "highlights": Array [], + "value": Array [ + "docs_microservice", + ], + }, + Object { + "constant": "/", + }, + Object { + "field": "haproxy.server_name", + "highlights": Array [], + "value": Array [ + "docs", + ], + }, + Object { + "constant": " \\"", + }, + Object { + "field": "haproxy.http.request.raw_request_line", + "highlights": Array [], + "value": Array [ + "GET /component---src-pages-index-js-4b15624544f97cf0bb8f.js HTTP/1.1", + ], + }, + Object { + "constant": "\\" ", + }, + Object { + "field": "http.response.status_code", + "highlights": Array [], + "value": Array [ + 304, + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "haproxy.http.request.time_wait_ms", + "highlights": Array [], + "value": Array [ + 0, + ], + }, + Object { + "constant": "/", + }, + Object { + "field": "event.duration", + "highlights": Array [], + "value": Array [ + 2000000, + ], + }, + Object { + "constant": "/", + }, + Object { + "field": "haproxy.connection_wait_time_ms", + "highlights": Array [], + "value": Array [ + 1, + ], + }, + Object { + "constant": "/", + }, + Object { + "field": "haproxy.http.request.time_wait_without_data_ms", + "highlights": Array [], + "value": Array [ + 0, + ], + }, + Object { + "constant": "/", + }, + Object { + "field": "event.duration", + "highlights": Array [], + "value": Array [ + 2000000, + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "haproxy.connections.active", + "highlights": Array [], + "value": Array [ + 6, + ], + }, + Object { + "constant": "/", + }, + Object { + "field": "haproxy.connections.frontend", + "highlights": Array [], + "value": Array [ + 6, + ], + }, + Object { + "constant": "/", + }, + Object { + "field": "haproxy.connections.backend", + "highlights": Array [], + "value": Array [ + 0, + ], + }, + Object { + "constant": "/", + }, + Object { + "field": "haproxy.connections.server", + "highlights": Array [], + "value": Array [ + 0, + ], + }, + Object { + "constant": "/", + }, + Object { + "field": "haproxy.connections.retries", + "highlights": Array [], + "value": Array [ + 0, + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "haproxy.server_queue", + "highlights": Array [], + "value": Array [ + 0, + ], + }, + Object { + "constant": "/", + }, + Object { + "field": "haproxy.backend_queue", + "highlights": Array [], + "value": Array [ + 0, + ], + }, + ] + `); }); }); describe('in pre-ECS format', () => { test('haproxy default log', () => { const flattenedDocument = { - 'event.dataset': 'haproxy.log', - 'fileset.module': 'haproxy', - 'fileset.name': 'log', - 'haproxy.client.ip': '1.2.3.4', - 'haproxy.client.port': '40780', - 'haproxy.destination.ip': '1.2.3.4', - 'haproxy.destination.port': '5000', - 'haproxy.frontend_name': 'main', - 'haproxy.geoip.continent_name': 'North America', - 'haproxy.geoip.country_iso_code': 'US', - 'haproxy.geoip.location.lat': 37.751, - 'haproxy.geoip.location.lon': -97.822, - 'haproxy.mode': 'HTTP', - 'haproxy.pid': '24551', - 'haproxy.process_name': 'haproxy', - 'haproxy.source': '1.2.3.4', - 'input.type': 'log', - offset: 0, - 'prospector.type': 'log', + 'event.dataset': ['haproxy.log'], + 'fileset.module': ['haproxy'], + 'fileset.name': ['log'], + 'haproxy.client.ip': ['1.2.3.4'], + 'haproxy.client.port': ['40780'], + 'haproxy.destination.ip': ['1.2.3.4'], + 'haproxy.destination.port': ['5000'], + 'haproxy.frontend_name': ['main'], + 'haproxy.geoip.continent_name': ['North America'], + 'haproxy.geoip.country_iso_code': ['US'], + 'haproxy.geoip.location.lat': [37.751], + 'haproxy.geoip.location.lon': [-97.822], + 'haproxy.mode': ['HTTP'], + 'haproxy.pid': ['24551'], + 'haproxy.process_name': ['haproxy'], + 'haproxy.source': ['1.2.3.4'], + 'input.type': ['log'], + offset: [0], + 'prospector.type': ['log'], }; expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` -Array [ - Object { - "constant": "[HAProxy] ", - }, - Object { - "field": "haproxy.client.ip", - "highlights": Array [], - "value": "1.2.3.4", - }, - Object { - "constant": ":", - }, - Object { - "field": "haproxy.client.port", - "highlights": Array [], - "value": "40780", - }, - Object { - "constant": " ", - }, - Object { - "field": "haproxy.frontend_name", - "highlights": Array [], - "value": "main", - }, -] -`); + Array [ + Object { + "constant": "[HAProxy] ", + }, + Object { + "field": "haproxy.client.ip", + "highlights": Array [], + "value": Array [ + "1.2.3.4", + ], + }, + Object { + "constant": ":", + }, + Object { + "field": "haproxy.client.port", + "highlights": Array [], + "value": Array [ + "40780", + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "haproxy.frontend_name", + "highlights": Array [], + "value": Array [ + "main", + ], + }, + ] + `); }); test('haproxy tcp log', () => { const flattenedDocument = { - 'event.dataset': 'haproxy.log', - 'fileset.module': 'haproxy', - 'fileset.name': 'log', - 'haproxy.backend_name': 'app', - 'haproxy.backend_queue': 0, - 'haproxy.bytes_read': 212, - 'haproxy.client.ip': '127.0.0.1', - 'haproxy.client.port': 40962, - 'haproxy.connection_wait_time_ms': -1, - 'haproxy.connections.active': 1, - 'haproxy.connections.backend': 0, - 'haproxy.connections.frontend': 1, - 'haproxy.connections.retries': 0, - 'haproxy.connections.server': 0, - 'haproxy.frontend_name': 'main', - 'haproxy.pid': 25457, - 'haproxy.process_name': 'haproxy', - 'haproxy.server_name': '', - 'haproxy.server_queue': 0, - 'haproxy.source': '127.0.0.1', - 'haproxy.tcp.processing_time_ms': 0, - 'haproxy.termination_state': 'SC', - 'haproxy.total_waiting_time_ms': -1, - 'input.type': 'log', - offset: 0, - 'prospector.type': 'log', + 'event.dataset': ['haproxy.log'], + 'fileset.module': ['haproxy'], + 'fileset.name': ['log'], + 'haproxy.backend_name': ['app'], + 'haproxy.backend_queue': [0], + 'haproxy.bytes_read': [212], + 'haproxy.client.ip': ['127.0.0.1'], + 'haproxy.client.port': [40962], + 'haproxy.connection_wait_time_ms': [-1], + 'haproxy.connections.active': [1], + 'haproxy.connections.backend': [0], + 'haproxy.connections.frontend': [1], + 'haproxy.connections.retries': [0], + 'haproxy.connections.server': [0], + 'haproxy.frontend_name': ['main'], + 'haproxy.pid': [25457], + 'haproxy.process_name': ['haproxy'], + 'haproxy.server_name': [''], + 'haproxy.server_queue': [0], + 'haproxy.source': ['127.0.0.1'], + 'haproxy.tcp.processing_time_ms': [0], + 'haproxy.termination_state': ['SC'], + 'haproxy.total_waiting_time_ms': [-1], + 'input.type': ['log'], + offset: [0], + 'prospector.type': ['log'], }; expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` -Array [ - Object { - "constant": "[HAProxy][tcp] ", - }, - Object { - "field": "haproxy.client.ip", - "highlights": Array [], - "value": "127.0.0.1", - }, - Object { - "constant": ":", - }, - Object { - "field": "haproxy.client.port", - "highlights": Array [], - "value": "40962", - }, - Object { - "constant": " ", - }, - Object { - "field": "haproxy.frontend_name", - "highlights": Array [], - "value": "main", - }, - Object { - "constant": " -> ", - }, - Object { - "field": "haproxy.backend_name", - "highlights": Array [], - "value": "app", - }, - Object { - "constant": "/", - }, - Object { - "field": "haproxy.server_name", - "highlights": Array [], - "value": "", - }, - Object { - "constant": " ", - }, - Object { - "field": "haproxy.connections.active", - "highlights": Array [], - "value": "1", - }, - Object { - "constant": "/", - }, - Object { - "field": "haproxy.connections.frontend", - "highlights": Array [], - "value": "1", - }, - Object { - "constant": "/", - }, - Object { - "field": "haproxy.connections.backend", - "highlights": Array [], - "value": "0", - }, - Object { - "constant": "/", - }, - Object { - "field": "haproxy.connections.server", - "highlights": Array [], - "value": "0", - }, - Object { - "constant": "/", - }, - Object { - "field": "haproxy.connections.retries", - "highlights": Array [], - "value": "0", - }, - Object { - "constant": " ", - }, - Object { - "field": "haproxy.server_queue", - "highlights": Array [], - "value": "0", - }, - Object { - "constant": "/", - }, - Object { - "field": "haproxy.backend_queue", - "highlights": Array [], - "value": "0", - }, -] -`); + Array [ + Object { + "constant": "[HAProxy][tcp] ", + }, + Object { + "field": "haproxy.client.ip", + "highlights": Array [], + "value": Array [ + "127.0.0.1", + ], + }, + Object { + "constant": ":", + }, + Object { + "field": "haproxy.client.port", + "highlights": Array [], + "value": Array [ + 40962, + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "haproxy.frontend_name", + "highlights": Array [], + "value": Array [ + "main", + ], + }, + Object { + "constant": " -> ", + }, + Object { + "field": "haproxy.backend_name", + "highlights": Array [], + "value": Array [ + "app", + ], + }, + Object { + "constant": "/", + }, + Object { + "field": "haproxy.server_name", + "highlights": Array [], + "value": Array [ + "", + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "haproxy.connections.active", + "highlights": Array [], + "value": Array [ + 1, + ], + }, + Object { + "constant": "/", + }, + Object { + "field": "haproxy.connections.frontend", + "highlights": Array [], + "value": Array [ + 1, + ], + }, + Object { + "constant": "/", + }, + Object { + "field": "haproxy.connections.backend", + "highlights": Array [], + "value": Array [ + 0, + ], + }, + Object { + "constant": "/", + }, + Object { + "field": "haproxy.connections.server", + "highlights": Array [], + "value": Array [ + 0, + ], + }, + Object { + "constant": "/", + }, + Object { + "field": "haproxy.connections.retries", + "highlights": Array [], + "value": Array [ + 0, + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "haproxy.server_queue", + "highlights": Array [], + "value": Array [ + 0, + ], + }, + Object { + "constant": "/", + }, + Object { + "field": "haproxy.backend_queue", + "highlights": Array [], + "value": Array [ + 0, + ], + }, + ] + `); }); test('haproxy http log', () => { const flattenedDocument = { - 'event.dataset': 'haproxy.log', - 'fileset.module': 'haproxy', - 'fileset.name': 'log', - 'haproxy.backend_name': 'docs_microservice', - 'haproxy.backend_queue': 0, - 'haproxy.bytes_read': 168, - 'haproxy.client.ip': '1.2.3.4', - 'haproxy.client.port': 38862, - 'haproxy.connection_wait_time_ms': 1, - 'haproxy.connections.active': 6, - 'haproxy.connections.backend': 0, - 'haproxy.connections.frontend': 6, - 'haproxy.connections.retries': 0, - 'haproxy.connections.server': 0, - 'haproxy.frontend_name': 'incoming~', - 'haproxy.geoip.continent_name': 'North America', - 'haproxy.geoip.country_iso_code': 'US', - 'haproxy.geoip.location.lat': 37.751, - 'haproxy.geoip.location.lon': -97.822, - 'haproxy.http.request.captured_cookie': '-', - 'haproxy.http.request.raw_request_line': + 'event.dataset': ['haproxy.log'], + 'fileset.module': ['haproxy'], + 'fileset.name': ['log'], + 'haproxy.backend_name': ['docs_microservice'], + 'haproxy.backend_queue': [0], + 'haproxy.bytes_read': [168], + 'haproxy.client.ip': ['1.2.3.4'], + 'haproxy.client.port': [38862], + 'haproxy.connection_wait_time_ms': [1], + 'haproxy.connections.active': [6], + 'haproxy.connections.backend': [0], + 'haproxy.connections.frontend': [6], + 'haproxy.connections.retries': [0], + 'haproxy.connections.server': [0], + 'haproxy.frontend_name': ['incoming~'], + 'haproxy.geoip.continent_name': ['North America'], + 'haproxy.geoip.country_iso_code': ['US'], + 'haproxy.geoip.location.lat': [37.751], + 'haproxy.geoip.location.lon': [-97.822], + 'haproxy.http.request.captured_cookie': ['-'], + 'haproxy.http.request.raw_request_line': [ 'GET /component---src-pages-index-js-4b15624544f97cf0bb8f.js HTTP/1.1', - 'haproxy.http.request.time_active_ms': 2, - 'haproxy.http.request.time_wait_ms': 0, - 'haproxy.http.request.time_wait_without_data_ms': 0, - 'haproxy.http.response.captured_cookie': '-', - 'haproxy.http.response.status_code': 304, - 'haproxy.pid': 32450, - 'haproxy.process_name': 'haproxy', - 'haproxy.server_name': 'docs', - 'haproxy.server_queue': 0, - 'haproxy.termination_state': '----', - 'haproxy.total_waiting_time_ms': 0, - 'input.type': 'log', - offset: 0, - 'prospector.type': 'log', + ], + 'haproxy.http.request.time_active_ms': [2], + 'haproxy.http.request.time_wait_ms': [0], + 'haproxy.http.request.time_wait_without_data_ms': [0], + 'haproxy.http.response.captured_cookie': ['-'], + 'haproxy.http.response.status_code': [304], + 'haproxy.pid': [32450], + 'haproxy.process_name': ['haproxy'], + 'haproxy.server_name': ['docs'], + 'haproxy.server_queue': [0], + 'haproxy.termination_state': ['----'], + 'haproxy.total_waiting_time_ms': [0], + 'input.type': ['log'], + offset: [0], + 'prospector.type': ['log'], }; expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` -Array [ - Object { - "constant": "[HAProxy][http] ", - }, - Object { - "field": "haproxy.client.ip", - "highlights": Array [], - "value": "1.2.3.4", - }, - Object { - "constant": ":", - }, - Object { - "field": "haproxy.client.port", - "highlights": Array [], - "value": "38862", - }, - Object { - "constant": " ", - }, - Object { - "field": "haproxy.frontend_name", - "highlights": Array [], - "value": "incoming~", - }, - Object { - "constant": " -> ", - }, - Object { - "field": "haproxy.backend_name", - "highlights": Array [], - "value": "docs_microservice", - }, - Object { - "constant": "/", - }, - Object { - "field": "haproxy.server_name", - "highlights": Array [], - "value": "docs", - }, - Object { - "constant": " \\"", - }, - Object { - "field": "haproxy.http.request.raw_request_line", - "highlights": Array [], - "value": "GET /component---src-pages-index-js-4b15624544f97cf0bb8f.js HTTP/1.1", - }, - Object { - "constant": "\\" ", - }, - Object { - "field": "haproxy.http.response.status_code", - "highlights": Array [], - "value": "304", - }, - Object { - "constant": " ", - }, - Object { - "field": "haproxy.http.request.time_wait_ms", - "highlights": Array [], - "value": "0", - }, - Object { - "constant": "/", - }, - Object { - "field": "haproxy.total_waiting_time_ms", - "highlights": Array [], - "value": "0", - }, - Object { - "constant": "/", - }, - Object { - "field": "haproxy.connection_wait_time_ms", - "highlights": Array [], - "value": "1", - }, - Object { - "constant": "/", - }, - Object { - "field": "haproxy.http.request.time_wait_without_data_ms", - "highlights": Array [], - "value": "0", - }, - Object { - "constant": "/", - }, - Object { - "field": "haproxy.http.request.time_active_ms", - "highlights": Array [], - "value": "2", - }, - Object { - "constant": " ", - }, - Object { - "field": "haproxy.connections.active", - "highlights": Array [], - "value": "6", - }, - Object { - "constant": "/", - }, - Object { - "field": "haproxy.connections.frontend", - "highlights": Array [], - "value": "6", - }, - Object { - "constant": "/", - }, - Object { - "field": "haproxy.connections.backend", - "highlights": Array [], - "value": "0", - }, - Object { - "constant": "/", - }, - Object { - "field": "haproxy.connections.server", - "highlights": Array [], - "value": "0", - }, - Object { - "constant": "/", - }, - Object { - "field": "haproxy.connections.retries", - "highlights": Array [], - "value": "0", - }, - Object { - "constant": " ", - }, - Object { - "field": "haproxy.server_queue", - "highlights": Array [], - "value": "0", - }, - Object { - "constant": "/", - }, - Object { - "field": "haproxy.backend_queue", - "highlights": Array [], - "value": "0", - }, -] -`); + Array [ + Object { + "constant": "[HAProxy][http] ", + }, + Object { + "field": "haproxy.client.ip", + "highlights": Array [], + "value": Array [ + "1.2.3.4", + ], + }, + Object { + "constant": ":", + }, + Object { + "field": "haproxy.client.port", + "highlights": Array [], + "value": Array [ + 38862, + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "haproxy.frontend_name", + "highlights": Array [], + "value": Array [ + "incoming~", + ], + }, + Object { + "constant": " -> ", + }, + Object { + "field": "haproxy.backend_name", + "highlights": Array [], + "value": Array [ + "docs_microservice", + ], + }, + Object { + "constant": "/", + }, + Object { + "field": "haproxy.server_name", + "highlights": Array [], + "value": Array [ + "docs", + ], + }, + Object { + "constant": " \\"", + }, + Object { + "field": "haproxy.http.request.raw_request_line", + "highlights": Array [], + "value": Array [ + "GET /component---src-pages-index-js-4b15624544f97cf0bb8f.js HTTP/1.1", + ], + }, + Object { + "constant": "\\" ", + }, + Object { + "field": "haproxy.http.response.status_code", + "highlights": Array [], + "value": Array [ + 304, + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "haproxy.http.request.time_wait_ms", + "highlights": Array [], + "value": Array [ + 0, + ], + }, + Object { + "constant": "/", + }, + Object { + "field": "haproxy.total_waiting_time_ms", + "highlights": Array [], + "value": Array [ + 0, + ], + }, + Object { + "constant": "/", + }, + Object { + "field": "haproxy.connection_wait_time_ms", + "highlights": Array [], + "value": Array [ + 1, + ], + }, + Object { + "constant": "/", + }, + Object { + "field": "haproxy.http.request.time_wait_without_data_ms", + "highlights": Array [], + "value": Array [ + 0, + ], + }, + Object { + "constant": "/", + }, + Object { + "field": "haproxy.http.request.time_active_ms", + "highlights": Array [], + "value": Array [ + 2, + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "haproxy.connections.active", + "highlights": Array [], + "value": Array [ + 6, + ], + }, + Object { + "constant": "/", + }, + Object { + "field": "haproxy.connections.frontend", + "highlights": Array [], + "value": Array [ + 6, + ], + }, + Object { + "constant": "/", + }, + Object { + "field": "haproxy.connections.backend", + "highlights": Array [], + "value": Array [ + 0, + ], + }, + Object { + "constant": "/", + }, + Object { + "field": "haproxy.connections.server", + "highlights": Array [], + "value": Array [ + 0, + ], + }, + Object { + "constant": "/", + }, + Object { + "field": "haproxy.connections.retries", + "highlights": Array [], + "value": Array [ + 0, + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "haproxy.server_queue", + "highlights": Array [], + "value": Array [ + 0, + ], + }, + Object { + "constant": "/", + }, + Object { + "field": "haproxy.backend_queue", + "highlights": Array [], + "value": Array [ + 0, + ], + }, + ] + `); }); }); }); diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_icinga.test.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_icinga.test.ts index 120137f15b883..00d282e1833d8 100644 --- a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_icinga.test.ts +++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_icinga.test.ts @@ -13,135 +13,155 @@ describe('Filebeat Rules', () => { describe('in pre-ECS format', () => { test('icinga debug log', () => { const flattenedDocument = { - '@timestamp': '2017-04-04T11:43:09.000Z', - 'event.dataset': 'icinga.debug', - 'fileset.module': 'icinga', - 'fileset.name': 'debug', - 'icinga.debug.facility': 'GraphiteWriter', - 'icinga.debug.message': + '@timestamp': ['2017-04-04T11:43:09.000Z'], + 'event.dataset': ['icinga.debug'], + 'fileset.module': ['icinga'], + 'fileset.name': ['debug'], + 'icinga.debug.facility': ['GraphiteWriter'], + 'icinga.debug.message': [ "Add to metric list:'icinga2.demo.services.procs.procs.perfdata.procs.warn 250 1491306189'.", - 'icinga.debug.severity': 'debug', - 'input.type': 'log', - offset: 0, - 'prospector.type': 'log', + ], + 'icinga.debug.severity': ['debug'], + 'input.type': ['log'], + offset: [0], + 'prospector.type': ['log'], }; expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` -Array [ - Object { - "constant": "[Icinga][", - }, - Object { - "field": "icinga.debug.facility", - "highlights": Array [], - "value": "GraphiteWriter", - }, - Object { - "constant": "][", - }, - Object { - "field": "icinga.debug.severity", - "highlights": Array [], - "value": "debug", - }, - Object { - "constant": "] ", - }, - Object { - "field": "icinga.debug.message", - "highlights": Array [], - "value": "Add to metric list:'icinga2.demo.services.procs.procs.perfdata.procs.warn 250 1491306189'.", - }, -] -`); + Array [ + Object { + "constant": "[Icinga][", + }, + Object { + "field": "icinga.debug.facility", + "highlights": Array [], + "value": Array [ + "GraphiteWriter", + ], + }, + Object { + "constant": "][", + }, + Object { + "field": "icinga.debug.severity", + "highlights": Array [], + "value": Array [ + "debug", + ], + }, + Object { + "constant": "] ", + }, + Object { + "field": "icinga.debug.message", + "highlights": Array [], + "value": Array [ + "Add to metric list:'icinga2.demo.services.procs.procs.perfdata.procs.warn 250 1491306189'.", + ], + }, + ] + `); }); test('icinga main log', () => { const flattenedDocument = { - '@timestamp': '2017-04-04T09:16:34.000Z', - 'event.dataset': 'icinga.main', - 'fileset.module': 'icinga', - 'fileset.name': 'main', - 'icinga.main.facility': 'Notification', - 'icinga.main.message': + '@timestamp': ['2017-04-04T09:16:34.000Z'], + 'event.dataset': ['icinga.main'], + 'fileset.module': ['icinga'], + 'fileset.name': ['main'], + 'icinga.main.facility': ['Notification'], + 'icinga.main.message': [ "Sending 'Recovery' notification 'demo!load!mail-icingaadmin for user 'on-call'", - 'icinga.main.severity': 'information', - 'input.type': 'log', - offset: 0, - 'prospector.type': 'log', + ], + 'icinga.main.severity': ['information'], + 'input.type': ['log'], + offset: [0], + 'prospector.type': ['log'], }; expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` -Array [ - Object { - "constant": "[Icinga][", - }, - Object { - "field": "icinga.main.facility", - "highlights": Array [], - "value": "Notification", - }, - Object { - "constant": "][", - }, - Object { - "field": "icinga.main.severity", - "highlights": Array [], - "value": "information", - }, - Object { - "constant": "] ", - }, - Object { - "field": "icinga.main.message", - "highlights": Array [], - "value": "Sending 'Recovery' notification 'demo!load!mail-icingaadmin for user 'on-call'", - }, -] -`); + Array [ + Object { + "constant": "[Icinga][", + }, + Object { + "field": "icinga.main.facility", + "highlights": Array [], + "value": Array [ + "Notification", + ], + }, + Object { + "constant": "][", + }, + Object { + "field": "icinga.main.severity", + "highlights": Array [], + "value": Array [ + "information", + ], + }, + Object { + "constant": "] ", + }, + Object { + "field": "icinga.main.message", + "highlights": Array [], + "value": Array [ + "Sending 'Recovery' notification 'demo!load!mail-icingaadmin for user 'on-call'", + ], + }, + ] + `); }); test('icinga startup log', () => { const flattenedDocument = { - 'event.dataset': 'icinga.startup', - 'fileset.module': 'icinga', - 'fileset.name': 'startup', - 'icinga.startup.facility': 'cli', - 'icinga.startup.message': 'Icinga application loader (version: r2.6.3-1)', - 'icinga.startup.severity': 'information', - 'input.type': 'log', - offset: 0, - 'prospector.type': 'log', + 'event.dataset': ['icinga.startup'], + 'fileset.module': ['icinga'], + 'fileset.name': ['startup'], + 'icinga.startup.facility': ['cli'], + 'icinga.startup.message': ['Icinga application loader (version: r2.6.3-1)'], + 'icinga.startup.severity': ['information'], + 'input.type': ['log'], + offset: [0], + 'prospector.type': ['log'], }; expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` -Array [ - Object { - "constant": "[Icinga][", - }, - Object { - "field": "icinga.startup.facility", - "highlights": Array [], - "value": "cli", - }, - Object { - "constant": "][", - }, - Object { - "field": "icinga.startup.severity", - "highlights": Array [], - "value": "information", - }, - Object { - "constant": "] ", - }, - Object { - "field": "icinga.startup.message", - "highlights": Array [], - "value": "Icinga application loader (version: r2.6.3-1)", - }, -] -`); + Array [ + Object { + "constant": "[Icinga][", + }, + Object { + "field": "icinga.startup.facility", + "highlights": Array [], + "value": Array [ + "cli", + ], + }, + Object { + "constant": "][", + }, + Object { + "field": "icinga.startup.severity", + "highlights": Array [], + "value": Array [ + "information", + ], + }, + Object { + "constant": "] ", + }, + Object { + "field": "icinga.startup.message", + "highlights": Array [], + "value": Array [ + "Icinga application loader (version: r2.6.3-1)", + ], + }, + ] + `); }); }); }); diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_iis.test.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_iis.test.ts index 72449c03b63a6..5238d5fe1da95 100644 --- a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_iis.test.ts +++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_iis.test.ts @@ -13,550 +13,622 @@ describe('Filebeat Rules', () => { describe('in ECS format', () => { test('iis access log', () => { const flattenedDocument = { - '@timestamp': '2018-01-01T10:11:12.000Z', - 'destination.address': '127.0.0.1', - 'destination.domain': 'example.com', - 'destination.ip': '127.0.0.1', - 'destination.port': 80, - 'ecs.version': '1.0.0-beta2', - 'event.dataset': 'iis.access', - 'event.duration': 789000000, - 'event.module': 'iis', - 'fileset.name': 'access', - 'http.request.body.bytes': 456, - 'http.request.method': 'GET', - 'http.request.referrer': '-', - 'http.response.body.bytes': 123, - 'http.response.status_code': 200, - 'http.version': '1.1', - 'iis.access.cookie': '-', - 'iis.access.server_name': 'MACHINE-NAME', - 'iis.access.site_name': 'W3SVC1', - 'iis.access.sub_status': 0, - 'iis.access.win32_status': 0, - 'input.type': 'log', - 'log.offset': 1204, - 'service.type': 'iis', - 'source.address': '85.181.35.98', - 'source.geo.city_name': 'Berlin', - 'source.geo.continent_name': 'Europe', - 'source.geo.country_iso_code': 'DE', - 'source.geo.location.lat': 52.4908, - 'source.geo.location.lon': 13.3275, - 'source.geo.region_iso_code': 'DE-BE', - 'source.geo.region_name': 'Land Berlin', - 'source.ip': '85.181.35.98', - 'url.path': '/', - 'url.query': 'q=100', - 'user.name': '-', - 'user_agent.device.name': 'Other', - 'user_agent.name': 'Chrome', - 'user_agent.original': + '@timestamp': ['2018-01-01T10:11:12.000Z'], + 'destination.address': ['127.0.0.1'], + 'destination.domain': ['example.com'], + 'destination.ip': ['127.0.0.1'], + 'destination.port': [80], + 'ecs.version': ['1.0.0-beta2'], + 'event.dataset': ['iis.access'], + 'event.duration': [789000000], + 'event.module': ['iis'], + 'fileset.name': ['access'], + 'http.request.body.bytes': [456], + 'http.request.method': ['GET'], + 'http.request.referrer': ['-'], + 'http.response.body.bytes': [123], + 'http.response.status_code': [200], + 'http.version': ['1.1'], + 'iis.access.cookie': ['-'], + 'iis.access.server_name': ['MACHINE-NAME'], + 'iis.access.site_name': ['W3SVC1'], + 'iis.access.sub_status': [0], + 'iis.access.win32_status': [0], + 'input.type': ['log'], + 'log.offset': [1204], + 'service.type': ['iis'], + 'source.address': ['85.181.35.98'], + 'source.geo.city_name': ['Berlin'], + 'source.geo.continent_name': ['Europe'], + 'source.geo.country_iso_code': ['DE'], + 'source.geo.location.lat': [52.4908], + 'source.geo.location.lon': [13.3275], + 'source.geo.region_iso_code': ['DE-BE'], + 'source.geo.region_name': ['Land Berlin'], + 'source.ip': ['85.181.35.98'], + 'url.path': ['/'], + 'url.query': ['q=100'], + 'user.name': ['-'], + 'user_agent.device.name': ['Other'], + 'user_agent.name': ['Chrome'], + 'user_agent.original': [ 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36', - 'user_agent.os.full': 'Mac OS X 10.14.0', - 'user_agent.os.name': 'Mac OS X', - 'user_agent.os.version': '10.14.0', - 'user_agent.version': '70.0.3538', + ], + 'user_agent.os.full': ['Mac OS X 10.14.0'], + 'user_agent.os.name': ['Mac OS X'], + 'user_agent.os.version': ['10.14.0'], + 'user_agent.version': ['70.0.3538'], }; expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` -Array [ - Object { - "constant": "[", - }, - Object { - "field": "event.module", - "highlights": Array [], - "value": "iis", - }, - Object { - "constant": "][access] ", - }, - Object { - "field": "source.ip", - "highlights": Array [], - "value": "85.181.35.98", - }, - Object { - "constant": " ", - }, - Object { - "field": "user.name", - "highlights": Array [], - "value": "-", - }, - Object { - "constant": " \\"", - }, - Object { - "field": "http.request.method", - "highlights": Array [], - "value": "GET", - }, - Object { - "constant": " ", - }, - Object { - "field": "url.path", - "highlights": Array [], - "value": "/", - }, - Object { - "constant": "?", - }, - Object { - "field": "url.query", - "highlights": Array [], - "value": "q=100", - }, - Object { - "constant": " HTTP/", - }, - Object { - "field": "http.version", - "highlights": Array [], - "value": "1.1", - }, - Object { - "constant": "\\" ", - }, - Object { - "field": "http.response.status_code", - "highlights": Array [], - "value": "200", - }, - Object { - "constant": " ", - }, - Object { - "field": "http.response.body.bytes", - "highlights": Array [], - "value": "123", - }, -] -`); + Array [ + Object { + "constant": "[", + }, + Object { + "field": "event.module", + "highlights": Array [], + "value": Array [ + "iis", + ], + }, + Object { + "constant": "][access] ", + }, + Object { + "field": "source.ip", + "highlights": Array [], + "value": Array [ + "85.181.35.98", + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "user.name", + "highlights": Array [], + "value": Array [ + "-", + ], + }, + Object { + "constant": " \\"", + }, + Object { + "field": "http.request.method", + "highlights": Array [], + "value": Array [ + "GET", + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "url.path", + "highlights": Array [], + "value": Array [ + "/", + ], + }, + Object { + "constant": "?", + }, + Object { + "field": "url.query", + "highlights": Array [], + "value": Array [ + "q=100", + ], + }, + Object { + "constant": " HTTP/", + }, + Object { + "field": "http.version", + "highlights": Array [], + "value": Array [ + "1.1", + ], + }, + Object { + "constant": "\\" ", + }, + Object { + "field": "http.response.status_code", + "highlights": Array [], + "value": Array [ + 200, + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "http.response.body.bytes", + "highlights": Array [], + "value": Array [ + 123, + ], + }, + ] + `); }); test('iis 7.5 access log', () => { const flattenedDocument = { - '@timestamp': '2018-08-28T18:24:25.000Z', - 'destination.address': '10.100.220.70', - 'destination.ip': '10.100.220.70', - 'destination.port': 80, - 'ecs.version': '1.0.0-beta2', - 'event.dataset': 'iis.access', - 'event.duration': 792000000, - 'event.module': 'iis', - 'fileset.name': 'access', - 'http.request.method': 'GET', - 'http.response.status_code': 404, - 'iis.access.sub_status': 4, - 'iis.access.win32_status': 2, - 'input.type': 'log', - 'log.offset': 244, - 'service.type': 'iis', - 'source.address': '10.100.118.31', - 'source.ip': '10.100.118.31', - 'url.path': '/', - 'url.query': 'q=100', - 'user.name': '-', - 'user_agent.device.name': 'Other', - 'user_agent.name': 'IE', - 'user_agent.original': - 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.3; WOW64; Trident/7.0; .NET4.0E; .NET4.0C; .NET CLR 3.5.30729; .NET CLR[ 2.0.50727](tel: 2050727); .NET CLR 3.0.30729)', - 'user_agent.os.name': 'Windows 8.1', - 'user_agent.version': '7.0', + '@timestamp': ['2018-08-28T18:24:25.000Z'], + 'destination.address': ['10.100.220.70'], + 'destination.ip': ['10.100.220.70'], + 'destination.port': [80], + 'ecs.version': ['1.0.0-beta2'], + 'event.dataset': ['iis.access'], + 'event.duration': [792000000], + 'event.module': ['iis'], + 'fileset.name': ['access'], + 'http.request.method': ['GET'], + 'http.response.status_code': [404], + 'iis.access.sub_status': [4], + 'iis.access.win32_status': [2], + 'input.type': ['log'], + 'log.offset': [244], + 'service.type': ['iis'], + 'source.address': ['10.100.118.31'], + 'source.ip': ['10.100.118.31'], + 'url.path': ['/'], + 'url.query': ['q=100'], + 'user.name': ['-'], + 'user_agent.device.name': ['Other'], + 'user_agent.name': ['IE'], + 'user_agent.original': [ + 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.3; WOW64; Trident/7.0; .NET4.0E; .NET4.0C; .NET CLR 3.5.30729; .NET CLR[ 2.0.50727](tel: [2050727); .NET CLR 3.0.30729)', + ], + 'user_agent.os.name': ['Windows 8.1'], + 'user_agent.version': ['7.0'], }; expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` -Array [ - Object { - "constant": "[", - }, - Object { - "field": "event.module", - "highlights": Array [], - "value": "iis", - }, - Object { - "constant": "][access] ", - }, - Object { - "field": "source.ip", - "highlights": Array [], - "value": "10.100.118.31", - }, - Object { - "constant": " ", - }, - Object { - "field": "user.name", - "highlights": Array [], - "value": "-", - }, - Object { - "constant": " \\"", - }, - Object { - "field": "http.request.method", - "highlights": Array [], - "value": "GET", - }, - Object { - "constant": " ", - }, - Object { - "field": "url.path", - "highlights": Array [], - "value": "/", - }, - Object { - "constant": "?", - }, - Object { - "field": "url.query", - "highlights": Array [], - "value": "q=100", - }, - Object { - "constant": " HTTP/", - }, - Object { - "field": "http.version", - "highlights": Array [], - "value": "undefined", - }, - Object { - "constant": "\\" ", - }, - Object { - "field": "http.response.status_code", - "highlights": Array [], - "value": "404", - }, - Object { - "constant": " ", - }, - Object { - "field": "http.response.body.bytes", - "highlights": Array [], - "value": "undefined", - }, -] -`); + Array [ + Object { + "constant": "[", + }, + Object { + "field": "event.module", + "highlights": Array [], + "value": Array [ + "iis", + ], + }, + Object { + "constant": "][access] ", + }, + Object { + "field": "source.ip", + "highlights": Array [], + "value": Array [ + "10.100.118.31", + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "user.name", + "highlights": Array [], + "value": Array [ + "-", + ], + }, + Object { + "constant": " \\"", + }, + Object { + "field": "http.request.method", + "highlights": Array [], + "value": Array [ + "GET", + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "url.path", + "highlights": Array [], + "value": Array [ + "/", + ], + }, + Object { + "constant": "?", + }, + Object { + "field": "url.query", + "highlights": Array [], + "value": Array [ + "q=100", + ], + }, + Object { + "constant": " HTTP/", + }, + Object { + "field": "http.version", + "highlights": Array [], + "value": Array [], + }, + Object { + "constant": "\\" ", + }, + Object { + "field": "http.response.status_code", + "highlights": Array [], + "value": Array [ + 404, + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "http.response.body.bytes", + "highlights": Array [], + "value": Array [], + }, + ] + `); }); test('iis error log', () => { const flattenedDocument = { - '@timestamp': '2018-01-01T08:09:10.000Z', - 'destination.address': '172.31.77.6', - 'destination.ip': '172.31.77.6', - 'destination.port': 80, - 'ecs.version': '1.0.0-beta2', - 'event.dataset': 'iis.error', - 'event.module': 'iis', - 'fileset.name': 'error', - 'http.request.method': 'GET', - 'http.response.status_code': 503, - 'http.version': '1.1', - 'iis.error.queue_name': '-', - 'iis.error.reason_phrase': 'ConnLimit', - 'input.type': 'log', - 'log.offset': 186, - 'service.type': 'iis', - 'source.address': '172.31.77.6', - 'source.ip': '172.31.77.6', - 'source.port': 2094, - 'url.original': '/qos/1kbfile.txt', + '@timestamp': ['2018-01-01T08:09:10.000Z'], + 'destination.address': ['172.31.77.6'], + 'destination.ip': ['172.31.77.6'], + 'destination.port': [80], + 'ecs.version': ['1.0.0-beta2'], + 'event.dataset': ['iis.error'], + 'event.module': ['iis'], + 'fileset.name': ['error'], + 'http.request.method': ['GET'], + 'http.response.status_code': [503], + 'http.version': ['1.1'], + 'iis.error.queue_name': ['-'], + 'iis.error.reason_phrase': ['ConnLimit'], + 'input.type': ['log'], + 'log.offset': [186], + 'service.type': ['iis'], + 'source.address': ['172.31.77.6'], + 'source.ip': ['172.31.77.6'], + 'source.port': [2094], + 'url.original': ['/qos/1kbfile.txt'], }; expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` -Array [ - Object { - "constant": "[iis][error] ", - }, - Object { - "field": "source.ip", - "highlights": Array [], - "value": "172.31.77.6", - }, - Object { - "constant": " ", - }, - Object { - "field": "iis.error.reason_phrase", - "highlights": Array [], - "value": "ConnLimit", - }, -] -`); + Array [ + Object { + "constant": "[iis][error] ", + }, + Object { + "field": "source.ip", + "highlights": Array [], + "value": Array [ + "172.31.77.6", + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "iis.error.reason_phrase", + "highlights": Array [], + "value": Array [ + "ConnLimit", + ], + }, + ] + `); }); }); describe('in pre-ECS format', () => { test('iis access log', () => { const flattenedDocument = { - '@timestamp': '2018-01-01T08:09:10.000Z', - 'event.dataset': 'iis.access', - 'fileset.module': 'iis', - 'fileset.name': 'access', - 'iis.access.geoip.city_name': 'Berlin', - 'iis.access.geoip.continent_name': 'Europe', - 'iis.access.geoip.country_iso_code': 'DE', - 'iis.access.geoip.location.lat': 52.4908, - 'iis.access.geoip.location.lon': 13.3275, - 'iis.access.geoip.region_iso_code': 'DE-BE', - 'iis.access.geoip.region_name': 'Land Berlin', - 'iis.access.method': 'GET', - 'iis.access.port': '80', - 'iis.access.query_string': 'q=100', - 'iis.access.referrer': '-', - 'iis.access.remote_ip': '85.181.35.98', - 'iis.access.request_time_ms': '123', - 'iis.access.response_code': '200', - 'iis.access.server_ip': '127.0.0.1', - 'iis.access.sub_status': '0', - 'iis.access.url': '/', - 'iis.access.user_agent.device': 'Other', - 'iis.access.user_agent.major': '57', - 'iis.access.user_agent.minor': '0', - 'iis.access.user_agent.name': 'Firefox', - 'iis.access.user_agent.original': + '@timestamp': ['2018-01-01T08:09:10.000Z'], + 'event.dataset': ['iis.access'], + 'fileset.module': ['iis'], + 'fileset.name': ['access'], + 'iis.access.geoip.city_name': ['Berlin'], + 'iis.access.geoip.continent_name': ['Europe'], + 'iis.access.geoip.country_iso_code': ['DE'], + 'iis.access.geoip.location.lat': [52.4908], + 'iis.access.geoip.location.lon': [13.3275], + 'iis.access.geoip.region_iso_code': ['DE-BE'], + 'iis.access.geoip.region_name': ['Land Berlin'], + 'iis.access.method': ['GET'], + 'iis.access.port': ['80'], + 'iis.access.query_string': ['q=100'], + 'iis.access.referrer': ['-'], + 'iis.access.remote_ip': ['85.181.35.98'], + 'iis.access.request_time_ms': ['123'], + 'iis.access.response_code': ['200'], + 'iis.access.server_ip': ['127.0.0.1'], + 'iis.access.sub_status': ['0'], + 'iis.access.url': ['/'], + 'iis.access.user_agent.device': ['Other'], + 'iis.access.user_agent.major': ['57'], + 'iis.access.user_agent.minor': ['0'], + 'iis.access.user_agent.name': ['Firefox'], + 'iis.access.user_agent.original': [ 'Mozilla/5.0+(Windows+NT+6.1;+Win64;+x64;+rv:57.0)+Gecko/20100101+Firefox/57.0', - 'iis.access.user_agent.os': 'Windows', - 'iis.access.user_agent.os_name': 'Windows', - 'iis.access.user_name': '-', - 'iis.access.win32_status': '0', - 'input.type': 'log', - offset: 257, - 'prospector.type': 'log', + ], + 'iis.access.user_agent.os': ['Windows'], + 'iis.access.user_agent.os_name': ['Windows'], + 'iis.access.user_name': ['-'], + 'iis.access.win32_status': ['0'], + 'input.type': ['log'], + offset: [257], + 'prospector.type': ['log'], }; expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` -Array [ - Object { - "constant": "[iis][access] ", - }, - Object { - "field": "iis.access.remote_ip", - "highlights": Array [], - "value": "85.181.35.98", - }, - Object { - "constant": " ", - }, - Object { - "field": "iis.access.user_name", - "highlights": Array [], - "value": "-", - }, - Object { - "constant": " \\"", - }, - Object { - "field": "iis.access.method", - "highlights": Array [], - "value": "GET", - }, - Object { - "constant": " ", - }, - Object { - "field": "iis.access.url", - "highlights": Array [], - "value": "/", - }, - Object { - "constant": " HTTP/", - }, - Object { - "field": "iis.access.http_version", - "highlights": Array [], - "value": "undefined", - }, - Object { - "constant": "\\" ", - }, - Object { - "field": "iis.access.response_code", - "highlights": Array [], - "value": "200", - }, - Object { - "constant": " ", - }, - Object { - "field": "iis.access.body_sent.bytes", - "highlights": Array [], - "value": "undefined", - }, -] -`); + Array [ + Object { + "constant": "[iis][access] ", + }, + Object { + "field": "iis.access.remote_ip", + "highlights": Array [], + "value": Array [ + "85.181.35.98", + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "iis.access.user_name", + "highlights": Array [], + "value": Array [ + "-", + ], + }, + Object { + "constant": " \\"", + }, + Object { + "field": "iis.access.method", + "highlights": Array [], + "value": Array [ + "GET", + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "iis.access.url", + "highlights": Array [], + "value": Array [ + "/", + ], + }, + Object { + "constant": " HTTP/", + }, + Object { + "field": "iis.access.http_version", + "highlights": Array [], + "value": Array [], + }, + Object { + "constant": "\\" ", + }, + Object { + "field": "iis.access.response_code", + "highlights": Array [], + "value": Array [ + "200", + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "iis.access.body_sent.bytes", + "highlights": Array [], + "value": Array [], + }, + ] + `); }); test('iis 7.5 access log', () => { const flattenedDocument = { - '@timestamp': '2018-08-28T18:24:25.000Z', - 'event.dataset': 'iis.access', - 'fileset.module': 'iis', - 'fileset.name': 'access', - 'iis.access.method': 'GET', - 'iis.access.port': '80', - 'iis.access.query_string': '-', - 'iis.access.remote_ip': '10.100.118.31', - 'iis.access.request_time_ms': '792', - 'iis.access.response_code': '404', - 'iis.access.server_ip': '10.100.220.70', - 'iis.access.sub_status': '4', - 'iis.access.url': '/', - 'iis.access.user_agent.device': 'Other', - 'iis.access.user_agent.name': 'Other', - 'iis.access.user_agent.original': + '@timestamp': ['2018-08-28T18:24:25.000Z'], + 'event.dataset': ['iis.access'], + 'fileset.module': ['iis'], + 'fileset.name': ['access'], + 'iis.access.method': ['GET'], + 'iis.access.port': ['80'], + 'iis.access.query_string': ['-'], + 'iis.access.remote_ip': ['10.100.118.31'], + 'iis.access.request_time_ms': ['792'], + 'iis.access.response_code': ['404'], + 'iis.access.server_ip': ['10.100.220.70'], + 'iis.access.sub_status': ['4'], + 'iis.access.url': ['/'], + 'iis.access.user_agent.device': ['Other'], + 'iis.access.user_agent.name': ['Other'], + 'iis.access.user_agent.original': [ 'Mozilla/4.0+(compatible;+MSIE+7.0;+Windows+NT+6.3;+WOW64;+Trident/7.0;+.NET4.0E;+.NET4.0C;+.NET+CLR+3.5.30729;+.NET+CLR[+2.0.50727](tel:+2050727);+.NET+CLR+3.0.30729)', - 'iis.access.user_agent.os': 'Windows', - 'iis.access.user_agent.os_name': 'Windows', - 'iis.access.user_name': '-', - 'iis.access.win32_status': '2', - 'input.type': 'log', - offset: 244, - 'prospector.type': 'log', + ], + 'iis.access.user_agent.os': ['Windows'], + 'iis.access.user_agent.os_name': ['Windows'], + 'iis.access.user_name': ['-'], + 'iis.access.win32_status': ['2'], + 'input.type': ['log'], + offset: [244], + 'prospector.type': ['log'], }; expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` -Array [ - Object { - "constant": "[iis][access] ", - }, - Object { - "field": "iis.access.remote_ip", - "highlights": Array [], - "value": "10.100.118.31", - }, - Object { - "constant": " ", - }, - Object { - "field": "iis.access.user_name", - "highlights": Array [], - "value": "-", - }, - Object { - "constant": " \\"", - }, - Object { - "field": "iis.access.method", - "highlights": Array [], - "value": "GET", - }, - Object { - "constant": " ", - }, - Object { - "field": "iis.access.url", - "highlights": Array [], - "value": "/", - }, - Object { - "constant": " HTTP/", - }, - Object { - "field": "iis.access.http_version", - "highlights": Array [], - "value": "undefined", - }, - Object { - "constant": "\\" ", - }, - Object { - "field": "iis.access.response_code", - "highlights": Array [], - "value": "404", - }, - Object { - "constant": " ", - }, - Object { - "field": "iis.access.body_sent.bytes", - "highlights": Array [], - "value": "undefined", - }, -] -`); + Array [ + Object { + "constant": "[iis][access] ", + }, + Object { + "field": "iis.access.remote_ip", + "highlights": Array [], + "value": Array [ + "10.100.118.31", + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "iis.access.user_name", + "highlights": Array [], + "value": Array [ + "-", + ], + }, + Object { + "constant": " \\"", + }, + Object { + "field": "iis.access.method", + "highlights": Array [], + "value": Array [ + "GET", + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "iis.access.url", + "highlights": Array [], + "value": Array [ + "/", + ], + }, + Object { + "constant": " HTTP/", + }, + Object { + "field": "iis.access.http_version", + "highlights": Array [], + "value": Array [], + }, + Object { + "constant": "\\" ", + }, + Object { + "field": "iis.access.response_code", + "highlights": Array [], + "value": Array [ + "404", + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "iis.access.body_sent.bytes", + "highlights": Array [], + "value": Array [], + }, + ] + `); }); test('iis error log', () => { const flattenedDocument = { - '@timestamp': '2018-01-01T08:09:10.000Z', - 'event.dataset': 'iis.error', - 'fileset.module': 'iis', - 'fileset.name': 'error', - 'iis.error.http_version': '1.1', - 'iis.error.method': 'GET', - 'iis.error.queue_name': '-', - 'iis.error.reason_phrase': 'ConnLimit', - 'iis.error.remote_ip': '172.31.77.6', - 'iis.error.remote_port': '2094', - 'iis.error.response_code': '503', - 'iis.error.server_ip': '172.31.77.6', - 'iis.error.server_port': '80', - 'iis.error.url': '/qos/1kbfile.txt', - 'input.type': 'log', - offset: 186, - 'prospector.type': 'log', + '@timestamp': ['2018-01-01T08:09:10.000Z'], + 'event.dataset': ['iis.error'], + 'fileset.module': ['iis'], + 'fileset.name': ['error'], + 'iis.error.http_version': ['1.1'], + 'iis.error.method': ['GET'], + 'iis.error.queue_name': ['-'], + 'iis.error.reason_phrase': ['ConnLimit'], + 'iis.error.remote_ip': ['172.31.77.6'], + 'iis.error.remote_port': ['2094'], + 'iis.error.response_code': ['503'], + 'iis.error.server_ip': ['172.31.77.6'], + 'iis.error.server_port': ['80'], + 'iis.error.url': ['/qos/1kbfile.txt'], + 'input.type': ['log'], + offset: [186], + 'prospector.type': ['log'], }; expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` -Array [ - Object { - "constant": "[iis][error] ", - }, - Object { - "field": "iis.error.remote_ip", - "highlights": Array [], - "value": "172.31.77.6", - }, - Object { - "constant": " \\"", - }, - Object { - "field": "iis.error.method", - "highlights": Array [], - "value": "GET", - }, - Object { - "constant": " ", - }, - Object { - "field": "iis.error.url", - "highlights": Array [], - "value": "/qos/1kbfile.txt", - }, - Object { - "constant": " HTTP/", - }, - Object { - "field": "iis.error.http_version", - "highlights": Array [], - "value": "1.1", - }, - Object { - "constant": "\\" ", - }, - Object { - "field": "iis.error.response_code", - "highlights": Array [], - "value": "503", - }, - Object { - "constant": " ", - }, - Object { - "field": "iis.error.reason_phrase", - "highlights": Array [], - "value": "ConnLimit", - }, -] -`); + Array [ + Object { + "constant": "[iis][error] ", + }, + Object { + "field": "iis.error.remote_ip", + "highlights": Array [], + "value": Array [ + "172.31.77.6", + ], + }, + Object { + "constant": " \\"", + }, + Object { + "field": "iis.error.method", + "highlights": Array [], + "value": Array [ + "GET", + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "iis.error.url", + "highlights": Array [], + "value": Array [ + "/qos/1kbfile.txt", + ], + }, + Object { + "constant": " HTTP/", + }, + Object { + "field": "iis.error.http_version", + "highlights": Array [], + "value": Array [ + "1.1", + ], + }, + Object { + "constant": "\\" ", + }, + Object { + "field": "iis.error.response_code", + "highlights": Array [], + "value": Array [ + "503", + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "iis.error.reason_phrase", + "highlights": Array [], + "value": Array [ + "ConnLimit", + ], + }, + ] + `); }); }); }); diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_kafka.test.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_kafka.test.ts index 19cb5f6e31118..4ea3cec8e91f6 100644 --- a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_kafka.test.ts +++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_kafka.test.ts @@ -13,48 +13,54 @@ describe('Filebeat Rules', () => { describe('in ECS format', () => { test('kafka log', () => { const flattenedDocument = { - '@timestamp': '2017-08-04T10:48:21.063Z', - 'ecs.version': '1.0.0-beta2', - 'event.dataset': 'kafka.log', - 'event.module': 'kafka', - 'fileset.name': 'log', - 'input.type': 'log', - 'kafka.log.class': 'kafka.controller.KafkaController', - 'kafka.log.component': 'Controller 0', - 'log.level': 'INFO', - 'log.offset': 131, - message: '0 successfully elected as the controller', - 'service.type': 'kafka', + '@timestamp': ['2017-08-04T10:48:21.063Z'], + 'ecs.version': ['1.0.0-beta2'], + 'event.dataset': ['kafka.log'], + 'event.module': ['kafka'], + 'fileset.name': ['log'], + 'input.type': ['log'], + 'kafka.log.class': ['kafka.controller.KafkaController'], + 'kafka.log.component': ['Controller 0'], + 'log.level': ['INFO'], + 'log.offset': [131], + message: ['0 successfully elected as the controller'], + 'service.type': ['kafka'], }; expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` -Array [ - Object { - "constant": "[", - }, - Object { - "field": "event.dataset", - "highlights": Array [], - "value": "kafka.log", - }, - Object { - "constant": "][", - }, - Object { - "field": "log.level", - "highlights": Array [], - "value": "INFO", - }, - Object { - "constant": "] ", - }, - Object { - "field": "message", - "highlights": Array [], - "value": "0 successfully elected as the controller", - }, -] -`); + Array [ + Object { + "constant": "[", + }, + Object { + "field": "event.dataset", + "highlights": Array [], + "value": Array [ + "kafka.log", + ], + }, + Object { + "constant": "][", + }, + Object { + "field": "log.level", + "highlights": Array [], + "value": Array [ + "INFO", + ], + }, + Object { + "constant": "] ", + }, + Object { + "field": "message", + "highlights": Array [], + "value": Array [ + "0 successfully elected as the controller", + ], + }, + ] + `); }); }); }); diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_logstash.test.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_logstash.test.ts index edc534d9c345f..022ea4921d9ad 100644 --- a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_logstash.test.ts +++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_logstash.test.ts @@ -13,194 +13,256 @@ describe('Filebeat Rules', () => { describe('in ECS format', () => { test('logstash log', () => { const flattenedDocument = { - '@timestamp': '2017-10-23T14:20:12.046Z', - 'ecs.version': '1.0.0-beta2', - 'event.dataset': 'logstash.log', - 'event.module': 'logstash', - 'fileset.name': 'log', - 'input.type': 'log', - 'log.level': 'INFO', - 'log.offset': 0, - 'logstash.log.module': 'logstash.modules.scaffold', - message: + '@timestamp': ['2017-10-23T14:20:12.046Z'], + 'ecs.version': ['1.0.0-beta2'], + 'event.dataset': ['logstash.log'], + 'event.module': ['logstash'], + 'fileset.name': ['log'], + 'input.type': ['log'], + 'log.level': ['INFO'], + 'log.offset': [0], + 'logstash.log.module': ['logstash.modules.scaffold'], + message: [ 'Initializing module {:module_name=>"fb_apache", :directory=>"/usr/share/logstash/modules/fb_apache/configuration"}', - 'service.type': 'logstash', + ], + 'service.type': ['logstash'], }; expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` -Array [ - Object { - "constant": "[", - }, - Object { - "field": "event.dataset", - "highlights": Array [], - "value": "logstash.log", - }, - Object { - "constant": "][", - }, - Object { - "field": "log.level", - "highlights": Array [], - "value": "INFO", - }, - Object { - "constant": "] ", - }, - Object { - "field": "message", - "highlights": Array [], - "value": "Initializing module {:module_name=>\\"fb_apache\\", :directory=>\\"/usr/share/logstash/modules/fb_apache/configuration\\"}", - }, -] -`); + Array [ + Object { + "constant": "[", + }, + Object { + "field": "event.dataset", + "highlights": Array [], + "value": Array [ + "logstash.log", + ], + }, + Object { + "constant": "][", + }, + Object { + "field": "log.level", + "highlights": Array [], + "value": Array [ + "INFO", + ], + }, + Object { + "constant": "] ", + }, + Object { + "field": "message", + "highlights": Array [], + "value": Array [ + "Initializing module {:module_name=>\\"fb_apache\\", :directory=>\\"/usr/share/logstash/modules/fb_apache/configuration\\"}", + ], + }, + ] + `); }); test('logstash slowlog', () => { const flattenedDocument = { - '@timestamp': '2017-10-30T09:57:58.243Z', - 'ecs.version': '1.0.0-beta2', - 'event.dataset': 'logstash.slowlog', - 'event.duration': 3027675106, - 'event.module': 'logstash', - 'fileset.name': 'slowlog', - 'input.type': 'log', - 'log.level': 'WARN', - 'log.offset': 0, - 'logstash.slowlog': { - event: - '"{\\"@version\\":\\"1\\",\\"@timestamp\\":\\"2017-10-30T13:57:55.130Z\\",\\"host\\":\\"sashimi\\",\\"sequence\\":0,\\"message\\":\\"Hello world!\\"}"', - module: 'slowlog.logstash.filters.sleep', - plugin_name: 'sleep', - plugin_params: - '{"time"=>3, "id"=>"e4e12a4e3082615c5427079bf4250dbfa338ebac10f8ea9912d7b98a14f56b8c"}', - plugin_type: 'filters', - took_in_millis: 3027, - }, - 'service.type': 'logstash', + '@timestamp': ['2017-10-30T09:57:58.243Z'], + 'ecs.version': ['1.0.0-beta2'], + 'event.dataset': ['logstash.slowlog'], + 'event.duration': [3027675106], + 'event.module': ['logstash'], + 'fileset.name': ['slowlog'], + 'input.type': ['log'], + 'log.level': ['WARN'], + 'log.offset': [0], + 'logstash.slowlog.event': [ + '"{\\"@version\\":\\"1\\",\\"@timestamp\\":\\"2017-10-30T13:57:55.130Z\\",\\"host\\":\\"sashimi\\",\\"sequence\\":0,\\"message\\":\\"Hello world!\\"}"', + ], + 'logstash.slowlog.module': ['slowlog.logstash.filters.sleep'], + 'logstash.slowlog.plugin_name': ['sleep'], + 'logstash.slowlog.plugin_params': [ + '{"time"=>3, "id"=>"e4e12a4e3082615c5427079bf4250dbfa338ebac10f8ea9912d7b98a14f56b8c"}', + ], + 'logstash.slowlog.plugin_type': ['filters'], + 'logstash.slowlog.took_in_millis': [3027], + 'service.type': ['logstash'], }; expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` -Array [ - Object { - "constant": "[Logstash][", - }, - Object { - "field": "log.level", - "highlights": Array [], - "value": "WARN", - }, - Object { - "constant": "] ", - }, - Object { - "field": "logstash.slowlog", - "highlights": Array [], - "value": "{\\"event\\":\\"\\\\\\"{\\\\\\\\\\\\\\"@version\\\\\\\\\\\\\\":\\\\\\\\\\\\\\"1\\\\\\\\\\\\\\",\\\\\\\\\\\\\\"@timestamp\\\\\\\\\\\\\\":\\\\\\\\\\\\\\"2017-10-30T13:57:55.130Z\\\\\\\\\\\\\\",\\\\\\\\\\\\\\"host\\\\\\\\\\\\\\":\\\\\\\\\\\\\\"sashimi\\\\\\\\\\\\\\",\\\\\\\\\\\\\\"sequence\\\\\\\\\\\\\\":0,\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\":\\\\\\\\\\\\\\"Hello world!\\\\\\\\\\\\\\"}\\\\\\"\\",\\"module\\":\\"slowlog.logstash.filters.sleep\\",\\"plugin_name\\":\\"sleep\\",\\"plugin_params\\":\\"{\\\\\\"time\\\\\\"=>3, \\\\\\"id\\\\\\"=>\\\\\\"e4e12a4e3082615c5427079bf4250dbfa338ebac10f8ea9912d7b98a14f56b8c\\\\\\"}\\",\\"plugin_type\\":\\"filters\\",\\"took_in_millis\\":3027}", - }, -] -`); + Array [ + Object { + "constant": "[Logstash][", + }, + Object { + "field": "log.level", + "highlights": Array [], + "value": Array [ + "WARN", + ], + }, + Object { + "constant": "] ", + }, + Object { + "field": "logstash.slowlog.event", + "highlights": Array [], + "value": Array [ + "\\"{\\\\\\"@version\\\\\\":\\\\\\"1\\\\\\",\\\\\\"@timestamp\\\\\\":\\\\\\"2017-10-30T13:57:55.130Z\\\\\\",\\\\\\"host\\\\\\":\\\\\\"sashimi\\\\\\",\\\\\\"sequence\\\\\\":0,\\\\\\"message\\\\\\":\\\\\\"Hello world!\\\\\\"}\\"", + ], + }, + Object { + "field": "logstash.slowlog.module", + "highlights": Array [], + "value": Array [ + "slowlog.logstash.filters.sleep", + ], + }, + Object { + "field": "logstash.slowlog.plugin_name", + "highlights": Array [], + "value": Array [ + "sleep", + ], + }, + Object { + "field": "logstash.slowlog.plugin_params", + "highlights": Array [], + "value": Array [ + "{\\"time\\"=>3, \\"id\\"=>\\"e4e12a4e3082615c5427079bf4250dbfa338ebac10f8ea9912d7b98a14f56b8c\\"}", + ], + }, + Object { + "field": "logstash.slowlog.plugin_type", + "highlights": Array [], + "value": Array [ + "filters", + ], + }, + Object { + "field": "logstash.slowlog.took_in_millis", + "highlights": Array [], + "value": Array [ + 3027, + ], + }, + ] + `); }); }); describe('in pre-ECS format', () => { test('logstash log', () => { const flattenedDocument = { - '@timestamp': '2017-10-23T14:20:12.046Z', - 'event.dataset': 'logstash.log', - 'fileset.module': 'logstash', - 'fileset.name': 'log', - 'input.type': 'log', - 'logstash.log.level': 'INFO', - 'logstash.log.message': + '@timestamp': ['2017-10-23T14:20:12.046Z'], + 'event.dataset': ['logstash.log'], + 'fileset.module': ['logstash'], + 'fileset.name': ['log'], + 'input.type': ['log'], + 'logstash.log.level': ['INFO'], + 'logstash.log.message': [ 'Initializing module {:module_name=>"fb_apache", :directory=>"/usr/share/logstash/modules/fb_apache/configuration"}', - 'logstash.log.module': 'logstash.modules.scaffold', - offset: 0, - 'prospector.type': 'log', + ], + 'logstash.log.module': ['logstash.modules.scaffold'], + offset: [0], + 'prospector.type': ['log'], }; expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` -Array [ - Object { - "constant": "[Logstash][", - }, - Object { - "field": "logstash.log.level", - "highlights": Array [], - "value": "INFO", - }, - Object { - "constant": "] ", - }, - Object { - "field": "logstash.log.module", - "highlights": Array [], - "value": "logstash.modules.scaffold", - }, - Object { - "constant": " - ", - }, - Object { - "field": "logstash.log.message", - "highlights": Array [], - "value": "Initializing module {:module_name=>\\"fb_apache\\", :directory=>\\"/usr/share/logstash/modules/fb_apache/configuration\\"}", - }, -] -`); + Array [ + Object { + "constant": "[Logstash][", + }, + Object { + "field": "logstash.log.level", + "highlights": Array [], + "value": Array [ + "INFO", + ], + }, + Object { + "constant": "] ", + }, + Object { + "field": "logstash.log.module", + "highlights": Array [], + "value": Array [ + "logstash.modules.scaffold", + ], + }, + Object { + "constant": " - ", + }, + Object { + "field": "logstash.log.message", + "highlights": Array [], + "value": Array [ + "Initializing module {:module_name=>\\"fb_apache\\", :directory=>\\"/usr/share/logstash/modules/fb_apache/configuration\\"}", + ], + }, + ] + `); }); test('logstash slowlog', () => { const flattenedDocument = { - '@timestamp': '2017-10-30T09:57:58.243Z', - 'event.dataset': 'logstash.slowlog', - 'fileset.module': 'logstash', - 'fileset.name': 'slowlog', - 'input.type': 'log', - 'logstash.slowlog.event': + '@timestamp': ['2017-10-30T09:57:58.243Z'], + 'event.dataset': ['logstash.slowlog'], + 'fileset.module': ['logstash'], + 'fileset.name': ['slowlog'], + 'input.type': ['log'], + 'logstash.slowlog.event': [ '"{\\"@version\\":\\"1\\",\\"@timestamp\\":\\"2017-10-30T13:57:55.130Z\\",\\"host\\":\\"sashimi\\",\\"sequence\\":0,\\"message\\":\\"Hello world!\\"}"', - 'logstash.slowlog.level': 'WARN', - 'logstash.slowlog.message': + ], + 'logstash.slowlog.level': ['WARN'], + 'logstash.slowlog.message': [ 'event processing time {:plugin_params=>{"time"=>3, "id"=>"e4e12a4e3082615c5427079bf4250dbfa338ebac10f8ea9912d7b98a14f56b8c"}, :took_in_nanos=>3027675106, :took_in_millis=>3027, :event=>"{\\"@version\\":\\"1\\",\\"@timestamp\\":\\"2017-10-30T13:57:55.130Z\\",\\"host\\":\\"sashimi\\",\\"sequence\\":0,\\"message\\":\\"Hello world!\\"}"}', - 'logstash.slowlog.module': 'slowlog.logstash.filters.sleep', - 'logstash.slowlog.plugin_name': 'sleep', - 'logstash.slowlog.plugin_params': + ], + 'logstash.slowlog.module': ['slowlog.logstash.filters.sleep'], + 'logstash.slowlog.plugin_name': ['sleep'], + 'logstash.slowlog.plugin_params': [ '{"time"=>3, "id"=>"e4e12a4e3082615c5427079bf4250dbfa338ebac10f8ea9912d7b98a14f56b8c"}', - 'logstash.slowlog.plugin_type': 'filters', - 'logstash.slowlog.took_in_millis': 3027, - 'logstash.slowlog.took_in_nanos': 3027675106, - offset: 0, - 'prospector.type': 'log', + ], + 'logstash.slowlog.plugin_type': ['filters'], + 'logstash.slowlog.took_in_millis': [3027], + 'logstash.slowlog.took_in_nanos': [3027675106], + offset: [0], + 'prospector.type': ['log'], }; expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` -Array [ - Object { - "constant": "[Logstash][", - }, - Object { - "field": "logstash.slowlog.level", - "highlights": Array [], - "value": "WARN", - }, - Object { - "constant": "] ", - }, - Object { - "field": "logstash.slowlog.module", - "highlights": Array [], - "value": "slowlog.logstash.filters.sleep", - }, - Object { - "constant": " - ", - }, - Object { - "field": "logstash.slowlog.message", - "highlights": Array [], - "value": "event processing time {:plugin_params=>{\\"time\\"=>3, \\"id\\"=>\\"e4e12a4e3082615c5427079bf4250dbfa338ebac10f8ea9912d7b98a14f56b8c\\"}, :took_in_nanos=>3027675106, :took_in_millis=>3027, :event=>\\"{\\\\\\"@version\\\\\\":\\\\\\"1\\\\\\",\\\\\\"@timestamp\\\\\\":\\\\\\"2017-10-30T13:57:55.130Z\\\\\\",\\\\\\"host\\\\\\":\\\\\\"sashimi\\\\\\",\\\\\\"sequence\\\\\\":0,\\\\\\"message\\\\\\":\\\\\\"Hello world!\\\\\\"}\\"}", - }, -] -`); + Array [ + Object { + "constant": "[Logstash][", + }, + Object { + "field": "logstash.slowlog.level", + "highlights": Array [], + "value": Array [ + "WARN", + ], + }, + Object { + "constant": "] ", + }, + Object { + "field": "logstash.slowlog.module", + "highlights": Array [], + "value": Array [ + "slowlog.logstash.filters.sleep", + ], + }, + Object { + "constant": " - ", + }, + Object { + "field": "logstash.slowlog.message", + "highlights": Array [], + "value": Array [ + "event processing time {:plugin_params=>{\\"time\\"=>3, \\"id\\"=>\\"e4e12a4e3082615c5427079bf4250dbfa338ebac10f8ea9912d7b98a14f56b8c\\"}, :took_in_nanos=>3027675106, :took_in_millis=>3027, :event=>\\"{\\\\\\"@version\\\\\\":\\\\\\"1\\\\\\",\\\\\\"@timestamp\\\\\\":\\\\\\"2017-10-30T13:57:55.130Z\\\\\\",\\\\\\"host\\\\\\":\\\\\\"sashimi\\\\\\",\\\\\\"sequence\\\\\\":0,\\\\\\"message\\\\\\":\\\\\\"Hello world!\\\\\\"}\\"}", + ], + }, + ] + `); }); }); }); diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_logstash.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_logstash.ts index 39b2058ca7cdb..9471465183670 100644 --- a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_logstash.ts +++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_logstash.ts @@ -34,7 +34,7 @@ export const filebeatLogstashRules = [ { // ECS when: { - exists: ['ecs.version', 'logstash.slowlog'], + all: [{ exists: ['ecs.version'] }, { existsPrefix: ['logstash.slowlog'] }], }, format: [ { @@ -47,7 +47,7 @@ export const filebeatLogstashRules = [ constant: '] ', }, { - field: 'logstash.slowlog', + fieldsPrefix: 'logstash.slowlog', }, ], }, diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_mongodb.test.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_mongodb.test.ts index 3df7ebec241cc..108ec29f550d2 100644 --- a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_mongodb.test.ts +++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_mongodb.test.ts @@ -13,40 +13,45 @@ describe('Filebeat Rules', () => { describe('in pre-ECS format', () => { test('mongodb log', () => { const flattenedDocument = { - '@timestamp': '2018-02-05T12:44:56.677Z', - 'event.dataset': 'mongodb.log', - 'fileset.module': 'mongodb', - 'fileset.name': 'log', - 'input.type': 'log', - 'mongodb.log.component': 'STORAGE', - 'mongodb.log.context': 'initandlisten', - 'mongodb.log.message': + '@timestamp': ['2018-02-05T12:44:56.677Z'], + 'event.dataset': ['mongodb.log'], + 'fileset.module': ['mongodb'], + 'fileset.name': ['log'], + 'input.type': ['log'], + 'mongodb.log.component': ['STORAGE'], + 'mongodb.log.context': ['initandlisten'], + 'mongodb.log.message': [ 'wiredtiger_open config: create,cache_size=8G,session_max=20000,eviction=(threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),file_manager=(close_idle_time=100000),checkpoint=(wait=60,log_size=2GB),statistics_log=(wait=0),', - 'mongodb.log.severity': 'I', - offset: 281, - 'prospector.type': 'log', + ], + 'mongodb.log.severity': ['I'], + offset: [281], + 'prospector.type': ['log'], }; expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` -Array [ - Object { - "constant": "[MongoDB][", - }, - Object { - "field": "mongodb.log.component", - "highlights": Array [], - "value": "STORAGE", - }, - Object { - "constant": "] ", - }, - Object { - "field": "mongodb.log.message", - "highlights": Array [], - "value": "wiredtiger_open config: create,cache_size=8G,session_max=20000,eviction=(threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),file_manager=(close_idle_time=100000),checkpoint=(wait=60,log_size=2GB),statistics_log=(wait=0),", - }, -] -`); + Array [ + Object { + "constant": "[MongoDB][", + }, + Object { + "field": "mongodb.log.component", + "highlights": Array [], + "value": Array [ + "STORAGE", + ], + }, + Object { + "constant": "] ", + }, + Object { + "field": "mongodb.log.message", + "highlights": Array [], + "value": Array [ + "wiredtiger_open config: create,cache_size=8G,session_max=20000,eviction=(threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),file_manager=(close_idle_time=100000),checkpoint=(wait=60,log_size=2GB),statistics_log=(wait=0),", + ], + }, + ] + `); }); }); }); diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_mysql.test.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_mysql.test.ts index 0329d53f92d08..99e83e9602480 100644 --- a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_mysql.test.ts +++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_mysql.test.ts @@ -13,139 +13,158 @@ describe('Filebeat Rules', () => { describe('in ECS format', () => { test('mysql error log', () => { const flattenedDocument = { - '@timestamp': '2016-12-09T12:08:33.335Z', - 'ecs.version': '1.0.0-beta2', - 'event.dataset': 'mysql.error', - 'event.module': 'mysql', - 'fileset.name': 'error', - 'input.type': 'log', - 'log.level': 'Warning', - 'log.offset': 92, - message: + '@timestamp': ['2016-12-09T12:08:33.335Z'], + 'ecs.version': ['1.0.0-beta2'], + 'event.dataset': ['mysql.error'], + 'event.module': ['mysql'], + 'fileset.name': ['error'], + 'input.type': ['log'], + 'log.level': ['Warning'], + 'log.offset': [92], + message: [ 'TIMESTAMP with implicit DEFAULT value is deprecated. Please use --explicit_defaults_for_timestamp server option (see documentation for more details).', - 'mysql.thread_id': 0, - 'service.type': 'mysql', + ], + 'mysql.thread_id': [0], + 'service.type': ['mysql'], }; expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` -Array [ - Object { - "constant": "[", - }, - Object { - "field": "event.dataset", - "highlights": Array [], - "value": "mysql.error", - }, - Object { - "constant": "][", - }, - Object { - "field": "log.level", - "highlights": Array [], - "value": "Warning", - }, - Object { - "constant": "] ", - }, - Object { - "field": "message", - "highlights": Array [], - "value": "TIMESTAMP with implicit DEFAULT value is deprecated. Please use --explicit_defaults_for_timestamp server option (see documentation for more details).", - }, -] -`); + Array [ + Object { + "constant": "[", + }, + Object { + "field": "event.dataset", + "highlights": Array [], + "value": Array [ + "mysql.error", + ], + }, + Object { + "constant": "][", + }, + Object { + "field": "log.level", + "highlights": Array [], + "value": Array [ + "Warning", + ], + }, + Object { + "constant": "] ", + }, + Object { + "field": "message", + "highlights": Array [], + "value": Array [ + "TIMESTAMP with implicit DEFAULT value is deprecated. Please use --explicit_defaults_for_timestamp server option (see documentation for more details).", + ], + }, + ] + `); }); test('mysql slowlog', () => { const flattenedDocument = { - '@timestamp': '2018-08-07T08:27:47.000Z', - 'ecs.version': '1.0.0-beta2', - 'event.dataset': 'mysql.slowlog', - 'event.duration': 4071491000, - 'event.module': 'mysql', - 'fileset.name': 'slowlog', - 'input.type': 'log', + '@timestamp': ['2018-08-07T08:27:47.000Z'], + 'ecs.version': ['1.0.0-beta2'], + 'event.dataset': ['mysql.slowlog'], + 'event.duration': [4071491000], + 'event.module': ['mysql'], + 'fileset.name': ['slowlog'], + 'input.type': ['log'], 'log.flags': ['multiline'], - 'log.offset': 526, - 'mysql.slowlog.current_user': 'appuser', - 'mysql.slowlog.lock_time.sec': 0.000212, - 'mysql.slowlog.query': + 'log.offset': [526], + 'mysql.slowlog.current_user': ['appuser'], + 'mysql.slowlog.lock_time.sec': [0.000212], + 'mysql.slowlog.query': [ 'SELECT mcu.mcu_guid, mcu.cus_guid, mcu.mcu_url, mcu.mcu_crawlelements, mcu.mcu_order, GROUP_CONCAT(mca.mca_guid SEPARATOR ";") as mca_guid\n FROM kat_mailcustomerurl mcu, kat_customer cus, kat_mailcampaign mca\n WHERE cus.cus_guid = mcu.cus_guid\n AND cus.pro_code = \'CYB\'\n AND cus.cus_offline = 0\n AND mca.cus_guid = cus.cus_guid\n AND (mcu.mcu_date IS NULL OR mcu.mcu_date < CURDATE())\n AND mcu.mcu_crawlelements IS NOT NULL\n GROUP BY mcu.mcu_guid\n ORDER BY mcu.mcu_order ASC\n LIMIT 1000;', - 'mysql.slowlog.rows_examined': 1489615, - 'mysql.slowlog.rows_sent': 1000, - 'mysql.thread_id': 10997316, - 'service.type': 'mysql', - 'source.domain': 'apphost', - 'source.ip': '1.1.1.1', - 'user.name': 'appuser', + ], + 'mysql.slowlog.rows_examined': [1489615], + 'mysql.slowlog.rows_sent': [1000], + 'mysql.thread_id': [10997316], + 'service.type': ['mysql'], + 'source.domain': ['apphost'], + 'source.ip': ['1.1.1.1'], + 'user.name': ['appuser'], }; expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` -Array [ - Object { - "constant": "[MySQL][slowlog] ", - }, - Object { - "field": "user.name", - "highlights": Array [], - "value": "appuser", - }, - Object { - "constant": "@", - }, - Object { - "field": "source.domain", - "highlights": Array [], - "value": "apphost", - }, - Object { - "constant": " [", - }, - Object { - "field": "source.ip", - "highlights": Array [], - "value": "1.1.1.1", - }, - Object { - "constant": "] ", - }, - Object { - "constant": " - ", - }, - Object { - "field": "event.duration", - "highlights": Array [], - "value": "4071491000", - }, - Object { - "constant": " ns - ", - }, - Object { - "field": "mysql.slowlog.query", - "highlights": Array [], - "value": "SELECT mcu.mcu_guid, mcu.cus_guid, mcu.mcu_url, mcu.mcu_crawlelements, mcu.mcu_order, GROUP_CONCAT(mca.mca_guid SEPARATOR \\";\\") as mca_guid - FROM kat_mailcustomerurl mcu, kat_customer cus, kat_mailcampaign mca - WHERE cus.cus_guid = mcu.cus_guid - AND cus.pro_code = 'CYB' - AND cus.cus_offline = 0 - AND mca.cus_guid = cus.cus_guid - AND (mcu.mcu_date IS NULL OR mcu.mcu_date < CURDATE()) - AND mcu.mcu_crawlelements IS NOT NULL - GROUP BY mcu.mcu_guid - ORDER BY mcu.mcu_order ASC - LIMIT 1000;", - }, -] -`); + Array [ + Object { + "constant": "[MySQL][slowlog] ", + }, + Object { + "field": "user.name", + "highlights": Array [], + "value": Array [ + "appuser", + ], + }, + Object { + "constant": "@", + }, + Object { + "field": "source.domain", + "highlights": Array [], + "value": Array [ + "apphost", + ], + }, + Object { + "constant": " [", + }, + Object { + "field": "source.ip", + "highlights": Array [], + "value": Array [ + "1.1.1.1", + ], + }, + Object { + "constant": "] ", + }, + Object { + "constant": " - ", + }, + Object { + "field": "event.duration", + "highlights": Array [], + "value": Array [ + 4071491000, + ], + }, + Object { + "constant": " ns - ", + }, + Object { + "field": "mysql.slowlog.query", + "highlights": Array [], + "value": Array [ + "SELECT mcu.mcu_guid, mcu.cus_guid, mcu.mcu_url, mcu.mcu_crawlelements, mcu.mcu_order, GROUP_CONCAT(mca.mca_guid SEPARATOR \\";\\") as mca_guid + FROM kat_mailcustomerurl mcu, kat_customer cus, kat_mailcampaign mca + WHERE cus.cus_guid = mcu.cus_guid + AND cus.pro_code = 'CYB' + AND cus.cus_offline = 0 + AND mca.cus_guid = cus.cus_guid + AND (mcu.mcu_date IS NULL OR mcu.mcu_date < CURDATE()) + AND mcu.mcu_crawlelements IS NOT NULL + GROUP BY mcu.mcu_guid + ORDER BY mcu.mcu_order ASC + LIMIT 1000;", + ], + }, + ] + `); }); }); describe('in pre-ECS format', () => { test('mysql error log', () => { const errorDoc = { - 'mysql.error.message': + 'mysql.error.message': [ "Access denied for user 'petclinicdd'@'47.153.152.234' (using password: YES)", + ], }; const message = format(errorDoc, {}); expect(message).toEqual([ @@ -155,18 +174,18 @@ Array [ { field: 'mysql.error.message', highlights: [], - value: "Access denied for user 'petclinicdd'@'47.153.152.234' (using password: YES)", + value: ["Access denied for user 'petclinicdd'@'47.153.152.234' (using password: YES)"], }, ]); }); test('mysql slow log', () => { const errorDoc = { - 'mysql.slowlog.query': 'select * from hosts', - 'mysql.slowlog.query_time.sec': 5, - 'mysql.slowlog.user': 'admin', - 'mysql.slowlog.ip': '192.168.1.42', - 'mysql.slowlog.host': 'webserver-01', + 'mysql.slowlog.query': ['select * from hosts'], + 'mysql.slowlog.query_time.sec': [5], + 'mysql.slowlog.user': ['admin'], + 'mysql.slowlog.ip': ['192.168.1.42'], + 'mysql.slowlog.host': ['webserver-01'], }; const message = format(errorDoc, {}); expect(message).toEqual([ @@ -176,7 +195,7 @@ Array [ { field: 'mysql.slowlog.user', highlights: [], - value: 'admin', + value: ['admin'], }, { constant: '@', @@ -184,7 +203,7 @@ Array [ { field: 'mysql.slowlog.host', highlights: [], - value: 'webserver-01', + value: ['webserver-01'], }, { constant: ' [', @@ -192,7 +211,7 @@ Array [ { field: 'mysql.slowlog.ip', highlights: [], - value: '192.168.1.42', + value: ['192.168.1.42'], }, { constant: '] ', @@ -203,7 +222,7 @@ Array [ { field: 'mysql.slowlog.query_time.sec', highlights: [], - value: '5', + value: [5], }, { constant: ' s - ', @@ -211,7 +230,7 @@ Array [ { field: 'mysql.slowlog.query', highlights: [], - value: 'select * from hosts', + value: ['select * from hosts'], }, ]); }); diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_nginx.test.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_nginx.test.ts index 0bc8ae1e907b8..62f48007eb0e3 100644 --- a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_nginx.test.ts +++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_nginx.test.ts @@ -13,252 +13,293 @@ describe('Filebeat Rules', () => { describe('in ECS format', () => { test('Nginx Access', () => { const flattenedDocument = { - '@timestamp': '2017-05-29T19:02:48.000Z', - 'ecs.version': '1.0.0-beta2', - 'event.dataset': 'nginx.access', - 'event.module': 'nginx', - 'fileset.name': 'access', - 'http.request.method': 'GET', - 'http.request.referrer': '-', - 'http.response.body.bytes': 612, - 'http.response.status_code': 404, - 'http.version': '1.1', - 'input.type': 'log', - 'log.offset': 183, - 'service.type': 'nginx', - 'source.ip': '172.17.0.1', - 'url.original': '/stringpatch', - 'user.name': '-', - 'user_agent.device': 'Other', - 'user_agent.major': '15', - 'user_agent.minor': '0', - 'user_agent.name': 'Firefox Alpha', - 'user_agent.original': + '@timestamp': ['2017-05-29T19:02:48.000Z'], + 'ecs.version': ['1.0.0-beta2'], + 'event.dataset': ['nginx.access'], + 'event.module': ['nginx'], + 'fileset.name': ['access'], + 'http.request.method': ['GET'], + 'http.request.referrer': ['-'], + 'http.response.body.bytes': [612], + 'http.response.status_code': [404], + 'http.version': ['1.1'], + 'input.type': ['log'], + 'log.offset': [183], + 'service.type': ['nginx'], + 'source.ip': ['172.17.0.1'], + 'url.original': ['/stringpatch'], + 'user.name': ['-'], + 'user_agent.device': ['Other'], + 'user_agent.major': ['15'], + 'user_agent.minor': ['0'], + 'user_agent.name': ['Firefox Alpha'], + 'user_agent.original': [ 'Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2', - 'user_agent.os.full_name': 'Windows 7', - 'user_agent.os.name': 'Windows 7', - 'user_agent.patch': 'a2', + ], + 'user_agent.os.full_name': ['Windows 7'], + 'user_agent.os.name': ['Windows 7'], + 'user_agent.patch': ['a2'], }; expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` -Array [ - Object { - "constant": "[", - }, - Object { - "field": "event.module", - "highlights": Array [], - "value": "nginx", - }, - Object { - "constant": "][access] ", - }, - Object { - "field": "source.ip", - "highlights": Array [], - "value": "172.17.0.1", - }, - Object { - "constant": " ", - }, - Object { - "field": "user.name", - "highlights": Array [], - "value": "-", - }, - Object { - "constant": " \\"", - }, - Object { - "field": "http.request.method", - "highlights": Array [], - "value": "GET", - }, - Object { - "constant": " ", - }, - Object { - "field": "url.original", - "highlights": Array [], - "value": "/stringpatch", - }, - Object { - "constant": " HTTP/", - }, - Object { - "field": "http.version", - "highlights": Array [], - "value": "1.1", - }, - Object { - "constant": "\\" ", - }, - Object { - "field": "http.response.status_code", - "highlights": Array [], - "value": "404", - }, - Object { - "constant": " ", - }, - Object { - "field": "http.response.body.bytes", - "highlights": Array [], - "value": "612", - }, -] -`); + Array [ + Object { + "constant": "[", + }, + Object { + "field": "event.module", + "highlights": Array [], + "value": Array [ + "nginx", + ], + }, + Object { + "constant": "][access] ", + }, + Object { + "field": "source.ip", + "highlights": Array [], + "value": Array [ + "172.17.0.1", + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "user.name", + "highlights": Array [], + "value": Array [ + "-", + ], + }, + Object { + "constant": " \\"", + }, + Object { + "field": "http.request.method", + "highlights": Array [], + "value": Array [ + "GET", + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "url.original", + "highlights": Array [], + "value": Array [ + "/stringpatch", + ], + }, + Object { + "constant": " HTTP/", + }, + Object { + "field": "http.version", + "highlights": Array [], + "value": Array [ + "1.1", + ], + }, + Object { + "constant": "\\" ", + }, + Object { + "field": "http.response.status_code", + "highlights": Array [], + "value": Array [ + 404, + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "http.response.body.bytes", + "highlights": Array [], + "value": Array [ + 612, + ], + }, + ] + `); }); test('Nginx Error', () => { const flattenedDocument = { - '@timestamp': '2016-10-25T14:49:34.000Z', - 'ecs.version': '1.0.0-beta2', - 'event.dataset': 'nginx.error', - 'event.module': 'nginx', - 'fileset.name': 'error', - 'input.type': 'log', - 'log.level': 'error', - 'log.offset': 0, - message: + '@timestamp': ['2016-10-25T14:49:34.000Z'], + 'ecs.version': ['1.0.0-beta2'], + 'event.dataset': ['nginx.error'], + 'event.module': ['nginx'], + 'fileset.name': ['error'], + 'input.type': ['log'], + 'log.level': ['error'], + 'log.offset': [0], + message: [ 'open() "/usr/local/Cellar/nginx/1.10.2_1/html/favicon.ico" failed (2: No such file or directory), client: 127.0.0.1, server: localhost, request: "GET /favicon.ico HTTP/1.1", host: "localhost:8080", referrer: "http://localhost:8080/"', - 'nginx.error.connection_id': 1, - 'process.pid': 54053, - 'process.thread.id': 0, - 'service.type': 'nginx', + ], + 'nginx.error.connection_id': [1], + 'process.pid': [54053], + 'process.thread.id': [0], + 'service.type': ['nginx'], }; expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` -Array [ - Object { - "constant": "[nginx]", - }, - Object { - "constant": "[", - }, - Object { - "field": "log.level", - "highlights": Array [], - "value": "error", - }, - Object { - "constant": "] ", - }, - Object { - "field": "message", - "highlights": Array [], - "value": "open() \\"/usr/local/Cellar/nginx/1.10.2_1/html/favicon.ico\\" failed (2: No such file or directory), client: 127.0.0.1, server: localhost, request: \\"GET /favicon.ico HTTP/1.1\\", host: \\"localhost:8080\\", referrer: \\"http://localhost:8080/\\"", - }, -] -`); + Array [ + Object { + "constant": "[nginx]", + }, + Object { + "constant": "[", + }, + Object { + "field": "log.level", + "highlights": Array [], + "value": Array [ + "error", + ], + }, + Object { + "constant": "] ", + }, + Object { + "field": "message", + "highlights": Array [], + "value": Array [ + "open() \\"/usr/local/Cellar/nginx/1.10.2_1/html/favicon.ico\\" failed (2: No such file or directory), client: 127.0.0.1, server: localhost, request: \\"GET /favicon.ico HTTP/1.1\\", host: \\"localhost:8080\\", referrer: \\"http://localhost:8080/\\"", + ], + }, + ] + `); }); }); describe('in pre-ECS format', () => { test('Nginx Access', () => { const flattenedDocument = { - 'nginx.access': true, - 'nginx.access.remote_ip': '192.168.1.42', - 'nginx.access.user_name': 'admin', - 'nginx.access.method': 'GET', - 'nginx.access.url': '/faq', - 'nginx.access.http_version': '1.1', - 'nginx.access.body_sent.bytes': 1024, - 'nginx.access.response_code': 200, + 'nginx.access': [true], + 'nginx.access.remote_ip': ['192.168.1.42'], + 'nginx.access.user_name': ['admin'], + 'nginx.access.method': ['GET'], + 'nginx.access.url': ['/faq'], + 'nginx.access.http_version': ['1.1'], + 'nginx.access.body_sent.bytes': [1024], + 'nginx.access.response_code': [200], }; expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` -Array [ - Object { - "constant": "[nginx][access] ", - }, - Object { - "field": "nginx.access.remote_ip", - "highlights": Array [], - "value": "192.168.1.42", - }, - Object { - "constant": " ", - }, - Object { - "field": "nginx.access.user_name", - "highlights": Array [], - "value": "admin", - }, - Object { - "constant": " \\"", - }, - Object { - "field": "nginx.access.method", - "highlights": Array [], - "value": "GET", - }, - Object { - "constant": " ", - }, - Object { - "field": "nginx.access.url", - "highlights": Array [], - "value": "/faq", - }, - Object { - "constant": " HTTP/", - }, - Object { - "field": "nginx.access.http_version", - "highlights": Array [], - "value": "1.1", - }, - Object { - "constant": "\\" ", - }, - Object { - "field": "nginx.access.response_code", - "highlights": Array [], - "value": "200", - }, - Object { - "constant": " ", - }, - Object { - "field": "nginx.access.body_sent.bytes", - "highlights": Array [], - "value": "1024", - }, -] -`); + Array [ + Object { + "constant": "[nginx][access] ", + }, + Object { + "field": "nginx.access.remote_ip", + "highlights": Array [], + "value": Array [ + "192.168.1.42", + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "nginx.access.user_name", + "highlights": Array [], + "value": Array [ + "admin", + ], + }, + Object { + "constant": " \\"", + }, + Object { + "field": "nginx.access.method", + "highlights": Array [], + "value": Array [ + "GET", + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "nginx.access.url", + "highlights": Array [], + "value": Array [ + "/faq", + ], + }, + Object { + "constant": " HTTP/", + }, + Object { + "field": "nginx.access.http_version", + "highlights": Array [], + "value": Array [ + "1.1", + ], + }, + Object { + "constant": "\\" ", + }, + Object { + "field": "nginx.access.response_code", + "highlights": Array [], + "value": Array [ + 200, + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "nginx.access.body_sent.bytes", + "highlights": Array [], + "value": Array [ + 1024, + ], + }, + ] + `); }); test('Nginx Error', () => { const flattenedDocument = { - 'nginx.error.message': + 'nginx.error.message': [ 'connect() failed (111: Connection refused) while connecting to upstream, client: 127.0.0.1, server: localhost, request: "GET /php-status?json= HTTP/1.1", upstream: "fastcgi://[::1]:9000", host: "localhost"', - 'nginx.error.level': 'error', + ], + 'nginx.error.level': ['error'], }; expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` -Array [ - Object { - "constant": "[nginx]", - }, - Object { - "constant": "[", - }, - Object { - "field": "nginx.error.level", - "highlights": Array [], - "value": "error", - }, - Object { - "constant": "] ", - }, - Object { - "field": "nginx.error.message", - "highlights": Array [], - "value": "connect() failed (111: Connection refused) while connecting to upstream, client: 127.0.0.1, server: localhost, request: \\"GET /php-status?json= HTTP/1.1\\", upstream: \\"fastcgi://[::1]:9000\\", host: \\"localhost\\"", - }, -] -`); + Array [ + Object { + "constant": "[nginx]", + }, + Object { + "constant": "[", + }, + Object { + "field": "nginx.error.level", + "highlights": Array [], + "value": Array [ + "error", + ], + }, + Object { + "constant": "] ", + }, + Object { + "field": "nginx.error.message", + "highlights": Array [], + "value": Array [ + "connect() failed (111: Connection refused) while connecting to upstream, client: 127.0.0.1, server: localhost, request: \\"GET /php-status?json= HTTP/1.1\\", upstream: \\"fastcgi://[::1]:9000\\", host: \\"localhost\\"", + ], + }, + ] + `); }); }); }); diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_osquery.test.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_osquery.test.ts index 8dc70053e2022..513a9a8fcd910 100644 --- a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_osquery.test.ts +++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_osquery.test.ts @@ -13,65 +13,139 @@ describe('Filebeat Rules', () => { describe('in pre-ECS format', () => { test('osquery result log', () => { const flattenedDocument = { - '@timestamp': '2017-12-28T14:40:08.000Z', - 'event.dataset': 'osquery.result', - 'fileset.module': 'osquery', - 'fileset.name': 'result', - 'input.type': 'log', - offset: 0, - 'osquery.result.action': 'removed', - 'osquery.result.calendar_time': 'Thu Dec 28 14:40:08 2017 UTC', - 'osquery.result.columns': { - blocks: '122061322', - blocks_available: '75966945', - blocks_free: '121274885', - blocks_size: '4096', - device: '/dev/disk1s4', - device_alias: '/dev/disk1s4', - flags: '345018372', - inodes: '9223372036854775807', - inodes_free: '9223372036854775804', - path: '/private/var/vm', - type: 'apfs', - }, - 'osquery.result.counter': '1', - 'osquery.result.decorations.host_uuid': '4AB2906D-5516-5794-AF54-86D1D7F533F3', - 'osquery.result.decorations.username': 'tsg', - 'osquery.result.epoch': '0', - 'osquery.result.host_identifier': '192-168-0-4.rdsnet.ro', - 'osquery.result.name': 'pack_it-compliance_mounts', - 'osquery.result.unix_time': '1514472008', - 'prospector.type': 'log', + '@timestamp': ['2017-12-28T14:40:08.000Z'], + 'event.dataset': ['osquery.result'], + 'fileset.module': ['osquery'], + 'fileset.name': ['result'], + 'input.type': ['log'], + offset: [0], + 'osquery.result.action': ['removed'], + 'osquery.result.calendar_time': ['Thu Dec 28 14:40:08 2017 UTC'], + 'osquery.result.columns.blocks': ['122061322'], + 'osquery.result.columns.blocks_available': ['75966945'], + 'osquery.result.columns.blocks_free': ['121274885'], + 'osquery.result.columns.blocks_size': ['4096'], + 'osquery.result.columns.device': ['/dev/disk1s4'], + 'osquery.result.columns.device_alias': ['/dev/disk1s4'], + 'osquery.result.columns.flags': ['345018372'], + 'osquery.result.columns.inodes': ['9223372036854775807'], + 'osquery.result.columns.inodes_free': ['9223372036854775804'], + 'osquery.result.columns.path': ['/private/var/vm'], + 'osquery.result.columns.type': ['apfs'], + 'osquery.result.counter': ['1'], + 'osquery.result.decorations.host_uuid': ['4AB2906D-5516-5794-AF54-86D1D7F533F3'], + 'osquery.result.decorations.username': ['tsg'], + 'osquery.result.epoch': ['0'], + 'osquery.result.host_identifier': ['192-168-0-4.rdsnet.ro'], + 'osquery.result.name': ['pack_it-compliance_mounts'], + 'osquery.result.unix_time': ['1514472008'], + 'prospector.type': ['log'], }; expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` -Array [ - Object { - "constant": "[Osquery][", - }, - Object { - "field": "osquery.result.action", - "highlights": Array [], - "value": "removed", - }, - Object { - "constant": "] ", - }, - Object { - "field": "osquery.result.host_identifier", - "highlights": Array [], - "value": "192-168-0-4.rdsnet.ro", - }, - Object { - "constant": " ", - }, - Object { - "field": "osquery.result.columns", - "highlights": Array [], - "value": "{\\"blocks\\":\\"122061322\\",\\"blocks_available\\":\\"75966945\\",\\"blocks_free\\":\\"121274885\\",\\"blocks_size\\":\\"4096\\",\\"device\\":\\"/dev/disk1s4\\",\\"device_alias\\":\\"/dev/disk1s4\\",\\"flags\\":\\"345018372\\",\\"inodes\\":\\"9223372036854775807\\",\\"inodes_free\\":\\"9223372036854775804\\",\\"path\\":\\"/private/var/vm\\",\\"type\\":\\"apfs\\"}", - }, -] -`); + Array [ + Object { + "constant": "[Osquery][", + }, + Object { + "field": "osquery.result.action", + "highlights": Array [], + "value": Array [ + "removed", + ], + }, + Object { + "constant": "] ", + }, + Object { + "field": "osquery.result.host_identifier", + "highlights": Array [], + "value": Array [ + "192-168-0-4.rdsnet.ro", + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "osquery.result.columns.blocks", + "highlights": Array [], + "value": Array [ + "122061322", + ], + }, + Object { + "field": "osquery.result.columns.blocks_available", + "highlights": Array [], + "value": Array [ + "75966945", + ], + }, + Object { + "field": "osquery.result.columns.blocks_free", + "highlights": Array [], + "value": Array [ + "121274885", + ], + }, + Object { + "field": "osquery.result.columns.blocks_size", + "highlights": Array [], + "value": Array [ + "4096", + ], + }, + Object { + "field": "osquery.result.columns.device", + "highlights": Array [], + "value": Array [ + "/dev/disk1s4", + ], + }, + Object { + "field": "osquery.result.columns.device_alias", + "highlights": Array [], + "value": Array [ + "/dev/disk1s4", + ], + }, + Object { + "field": "osquery.result.columns.flags", + "highlights": Array [], + "value": Array [ + "345018372", + ], + }, + Object { + "field": "osquery.result.columns.inodes", + "highlights": Array [], + "value": Array [ + "9223372036854775807", + ], + }, + Object { + "field": "osquery.result.columns.inodes_free", + "highlights": Array [], + "value": Array [ + "9223372036854775804", + ], + }, + Object { + "field": "osquery.result.columns.path", + "highlights": Array [], + "value": Array [ + "/private/var/vm", + ], + }, + Object { + "field": "osquery.result.columns.type", + "highlights": Array [], + "value": Array [ + "apfs", + ], + }, + ] + `); }); }); }); diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_osquery.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_osquery.ts index b3a6ee8c5cb47..638fd01d50c90 100644 --- a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_osquery.ts +++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_osquery.ts @@ -27,7 +27,7 @@ export const filebeatOsqueryRules = [ constant: ' ', }, { - field: 'osquery.result.columns', + fieldsPrefix: 'osquery.result.columns', }, ], }, diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_traefik.test.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_traefik.test.ts index b19124558fdd0..d5d16f26a282a 100644 --- a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_traefik.test.ts +++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_traefik.test.ts @@ -13,112 +13,129 @@ describe('Filebeat Rules', () => { describe('in pre-ECS format', () => { test('traefik access log', () => { const flattenedDocument = { - '@timestamp': '2017-10-02T20:22:08.000Z', - 'event.dataset': 'traefik.access', - 'fileset.module': 'traefik', - 'fileset.name': 'access', - 'input.type': 'log', - offset: 280, - 'prospector.type': 'log', - 'traefik.access.backend_url': 'http://172.19.0.3:5601', - 'traefik.access.body_sent.bytes': 0, - 'traefik.access.duration': 3, - 'traefik.access.frontend_name': 'Host-host1', - 'traefik.access.geoip.city_name': 'Berlin', - 'traefik.access.geoip.continent_name': 'Europe', - 'traefik.access.geoip.country_iso_code': 'DE', - 'traefik.access.geoip.location.lat': 52.4908, - 'traefik.access.geoip.location.lon': 13.3275, - 'traefik.access.geoip.region_iso_code': 'DE-BE', - 'traefik.access.geoip.region_name': 'Land Berlin', - 'traefik.access.http_version': '1.1', - 'traefik.access.method': 'GET', - 'traefik.access.referrer': 'http://example.com/login', - 'traefik.access.remote_ip': '85.181.35.98', - 'traefik.access.request_count': 271, - 'traefik.access.response_code': '304', - 'traefik.access.url': '/ui/favicons/favicon.ico', - 'traefik.access.user_agent.device': 'Other', - 'traefik.access.user_agent.major': '61', - 'traefik.access.user_agent.minor': '0', - 'traefik.access.user_agent.name': 'Chrome', - 'traefik.access.user_agent.original': + '@timestamp': ['2017-10-02T20:22:08.000Z'], + 'event.dataset': ['traefik.access'], + 'fileset.module': ['traefik'], + 'fileset.name': ['access'], + 'input.type': ['log'], + offset: [280], + 'prospector.type': ['log'], + 'traefik.access.backend_url': ['http://172.19.0.3:5601'], + 'traefik.access.body_sent.bytes': [0], + 'traefik.access.duration': [3], + 'traefik.access.frontend_name': ['Host-host1'], + 'traefik.access.geoip.city_name': ['Berlin'], + 'traefik.access.geoip.continent_name': ['Europe'], + 'traefik.access.geoip.country_iso_code': ['DE'], + 'traefik.access.geoip.location.lat': [52.4908], + 'traefik.access.geoip.location.lon': [13.3275], + 'traefik.access.geoip.region_iso_code': ['DE-BE'], + 'traefik.access.geoip.region_name': ['Land Berlin'], + 'traefik.access.http_version': ['1.1'], + 'traefik.access.method': ['GET'], + 'traefik.access.referrer': ['http://example.com/login'], + 'traefik.access.remote_ip': ['85.181.35.98'], + 'traefik.access.request_count': [271], + 'traefik.access.response_code': ['304'], + 'traefik.access.url': ['/ui/favicons/favicon.ico'], + 'traefik.access.user_agent.device': ['Other'], + 'traefik.access.user_agent.major': ['61'], + 'traefik.access.user_agent.minor': ['0'], + 'traefik.access.user_agent.name': ['Chrome'], + 'traefik.access.user_agent.original': [ 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36', - 'traefik.access.user_agent.os': 'Linux', - 'traefik.access.user_agent.os_name': 'Linux', - 'traefik.access.user_agent.patch': '3163', - 'traefik.access.user_identifier': '-', - 'traefik.access.user_name': '-', + ], + 'traefik.access.user_agent.os': ['Linux'], + 'traefik.access.user_agent.os_name': ['Linux'], + 'traefik.access.user_agent.patch': ['3163'], + 'traefik.access.user_identifier': ['-'], + 'traefik.access.user_name': ['-'], }; expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` -Array [ - Object { - "constant": "[traefik][access] ", - }, - Object { - "field": "traefik.access.remote_ip", - "highlights": Array [], - "value": "85.181.35.98", - }, - Object { - "constant": " ", - }, - Object { - "field": "traefik.access.frontend_name", - "highlights": Array [], - "value": "Host-host1", - }, - Object { - "constant": " -> ", - }, - Object { - "field": "traefik.access.backend_url", - "highlights": Array [], - "value": "http://172.19.0.3:5601", - }, - Object { - "constant": " \\"", - }, - Object { - "field": "traefik.access.method", - "highlights": Array [], - "value": "GET", - }, - Object { - "constant": " ", - }, - Object { - "field": "traefik.access.url", - "highlights": Array [], - "value": "/ui/favicons/favicon.ico", - }, - Object { - "constant": " HTTP/", - }, - Object { - "field": "traefik.access.http_version", - "highlights": Array [], - "value": "1.1", - }, - Object { - "constant": "\\" ", - }, - Object { - "field": "traefik.access.response_code", - "highlights": Array [], - "value": "304", - }, - Object { - "constant": " ", - }, - Object { - "field": "traefik.access.body_sent.bytes", - "highlights": Array [], - "value": "0", - }, -] -`); + Array [ + Object { + "constant": "[traefik][access] ", + }, + Object { + "field": "traefik.access.remote_ip", + "highlights": Array [], + "value": Array [ + "85.181.35.98", + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "traefik.access.frontend_name", + "highlights": Array [], + "value": Array [ + "Host-host1", + ], + }, + Object { + "constant": " -> ", + }, + Object { + "field": "traefik.access.backend_url", + "highlights": Array [], + "value": Array [ + "http://172.19.0.3:5601", + ], + }, + Object { + "constant": " \\"", + }, + Object { + "field": "traefik.access.method", + "highlights": Array [], + "value": Array [ + "GET", + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "traefik.access.url", + "highlights": Array [], + "value": Array [ + "/ui/favicons/favicon.ico", + ], + }, + Object { + "constant": " HTTP/", + }, + Object { + "field": "traefik.access.http_version", + "highlights": Array [], + "value": Array [ + "1.1", + ], + }, + Object { + "constant": "\\" ", + }, + Object { + "field": "traefik.access.response_code", + "highlights": Array [], + "value": Array [ + "304", + ], + }, + Object { + "constant": " ", + }, + Object { + "field": "traefik.access.body_sent.bytes", + "highlights": Array [], + "value": Array [ + 0, + ], + }, + ] + `); }); }); }); diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/generic.test.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/generic.test.ts index d168273626cfa..eba0563be3aaa 100644 --- a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/generic.test.ts +++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/generic.test.ts @@ -15,154 +15,174 @@ describe('Generic Rules', () => { describe('configurable message rules', () => { test('includes the event.dataset and log.level if present', () => { const flattenedDocument = { - '@timestamp': '2016-12-26T16:22:13.000Z', - 'event.dataset': 'generic.test', - 'log.level': 'TEST_LEVEL', - first_generic_message: 'TEST_MESSAGE', + '@timestamp': ['2016-12-26T16:22:13.000Z'], + 'event.dataset': ['generic.test'], + 'log.level': ['TEST_LEVEL'], + first_generic_message: ['TEST_MESSAGE'], }; const highlights = { first_generic_message: ['TEST'], }; expect(format(flattenedDocument, highlights)).toMatchInlineSnapshot(` -Array [ - Object { - "constant": "[", - }, - Object { - "field": "event.dataset", - "highlights": Array [], - "value": "generic.test", - }, - Object { - "constant": "][", - }, - Object { - "field": "log.level", - "highlights": Array [], - "value": "TEST_LEVEL", - }, - Object { - "constant": "] ", - }, - Object { - "field": "first_generic_message", - "highlights": Array [ - "TEST", - ], - "value": "TEST_MESSAGE", - }, -] -`); + Array [ + Object { + "constant": "[", + }, + Object { + "field": "event.dataset", + "highlights": Array [], + "value": Array [ + "generic.test", + ], + }, + Object { + "constant": "][", + }, + Object { + "field": "log.level", + "highlights": Array [], + "value": Array [ + "TEST_LEVEL", + ], + }, + Object { + "constant": "] ", + }, + Object { + "field": "first_generic_message", + "highlights": Array [ + "TEST", + ], + "value": Array [ + "TEST_MESSAGE", + ], + }, + ] + `); }); test('includes the log.level if present', () => { const flattenedDocument = { - '@timestamp': '2016-12-26T16:22:13.000Z', - 'log.level': 'TEST_LEVEL', - first_generic_message: 'TEST_MESSAGE', + '@timestamp': ['2016-12-26T16:22:13.000Z'], + 'log.level': ['TEST_LEVEL'], + first_generic_message: ['TEST_MESSAGE'], }; expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` -Array [ - Object { - "constant": "[", - }, - Object { - "field": "log.level", - "highlights": Array [], - "value": "TEST_LEVEL", - }, - Object { - "constant": "] ", - }, - Object { - "field": "first_generic_message", - "highlights": Array [], - "value": "TEST_MESSAGE", - }, -] -`); + Array [ + Object { + "constant": "[", + }, + Object { + "field": "log.level", + "highlights": Array [], + "value": Array [ + "TEST_LEVEL", + ], + }, + Object { + "constant": "] ", + }, + Object { + "field": "first_generic_message", + "highlights": Array [], + "value": Array [ + "TEST_MESSAGE", + ], + }, + ] + `); }); test('includes the message', () => { const firstFlattenedDocument = { - '@timestamp': '2016-12-26T16:22:13.000Z', - first_generic_message: 'FIRST_TEST_MESSAGE', + '@timestamp': ['2016-12-26T16:22:13.000Z'], + first_generic_message: ['FIRST_TEST_MESSAGE'], }; expect(format(firstFlattenedDocument, {})).toMatchInlineSnapshot(` -Array [ - Object { - "field": "first_generic_message", - "highlights": Array [], - "value": "FIRST_TEST_MESSAGE", - }, -] -`); + Array [ + Object { + "field": "first_generic_message", + "highlights": Array [], + "value": Array [ + "FIRST_TEST_MESSAGE", + ], + }, + ] + `); const secondFlattenedDocument = { - '@timestamp': '2016-12-26T16:22:13.000Z', - second_generic_message: 'SECOND_TEST_MESSAGE', + '@timestamp': ['2016-12-26T16:22:13.000Z'], + second_generic_message: ['SECOND_TEST_MESSAGE'], }; expect(format(secondFlattenedDocument, {})).toMatchInlineSnapshot(` -Array [ - Object { - "field": "second_generic_message", - "highlights": Array [], - "value": "SECOND_TEST_MESSAGE", - }, -] -`); + Array [ + Object { + "field": "second_generic_message", + "highlights": Array [], + "value": Array [ + "SECOND_TEST_MESSAGE", + ], + }, + ] + `); }); }); describe('log.original fallback', () => { test('includes the event.dataset if present', () => { const flattenedDocument = { - '@timestamp': '2016-12-26T16:22:13.000Z', - 'event.dataset': 'generic.test', - 'log.original': 'TEST_MESSAGE', + '@timestamp': ['2016-12-26T16:22:13.000Z'], + 'event.dataset': ['generic.test'], + 'log.original': ['TEST_MESSAGE'], }; expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` -Array [ - Object { - "constant": "[", - }, - Object { - "field": "event.dataset", - "highlights": Array [], - "value": "generic.test", - }, - Object { - "constant": "] ", - }, - Object { - "field": "log.original", - "highlights": Array [], - "value": "TEST_MESSAGE", - }, -] -`); + Array [ + Object { + "constant": "[", + }, + Object { + "field": "event.dataset", + "highlights": Array [], + "value": Array [ + "generic.test", + ], + }, + Object { + "constant": "] ", + }, + Object { + "field": "log.original", + "highlights": Array [], + "value": Array [ + "TEST_MESSAGE", + ], + }, + ] + `); }); test('includes the original message', () => { const flattenedDocument = { - '@timestamp': '2016-12-26T16:22:13.000Z', - 'log.original': 'TEST_MESSAGE', + '@timestamp': ['2016-12-26T16:22:13.000Z'], + 'log.original': ['TEST_MESSAGE'], }; expect(format(flattenedDocument, {})).toMatchInlineSnapshot(` -Array [ - Object { - "field": "log.original", - "highlights": Array [], - "value": "TEST_MESSAGE", - }, -] -`); + Array [ + Object { + "field": "log.original", + "highlights": Array [], + "value": Array [ + "TEST_MESSAGE", + ], + }, + ] + `); }); }); }); diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/helpers.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/helpers.ts index 9a6fa30e17e89..e925fab75d984 100644 --- a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/helpers.ts +++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/helpers.ts @@ -10,3 +10,10 @@ export const labelField = (label: string, field: string) => [ { constant: '=' }, { field }, ]; + +export const labelFieldsPrefix = (label: string, fieldsPrefix: string) => [ + { constant: ' ' }, + { constant: label }, + { constant: '=' }, + { fieldsPrefix }, +]; diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/convert_document_source_to_log_item_fields.test.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/convert_document_source_to_log_item_fields.test.ts index 98d1e2cd89b01..7b79a1bf0386a 100644 --- a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/convert_document_source_to_log_item_fields.test.ts +++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/convert_document_source_to_log_item_fields.test.ts @@ -4,66 +4,62 @@ * you may not use this file except in compliance with the Elastic License. */ -import { convertDocumentSourceToLogItemFields } from './convert_document_source_to_log_item_fields'; +import { convertESFieldsToLogItemFields } from './convert_document_source_to_log_item_fields'; -describe('convertDocumentSourceToLogItemFields', () => { - test('should convert document', () => { - const doc = { - agent: { - hostname: 'demo-stack-client-01', - id: '7adef8b6-2ab7-45cd-a0d5-b3baad735f1b', - type: 'filebeat', - ephemeral_id: 'a0c8164b-3564-4e32-b0bf-f4db5a7ae566', - version: '7.0.0', - }, +describe('convertESFieldsToLogItemFields', () => { + test('Converts the fields collection to LogItemFields', () => { + const esFields = { + 'agent.hostname': ['demo-stack-client-01'], + 'agent.id': ['7adef8b6-2ab7-45cd-a0d5-b3baad735f1b'], + 'agent.type': ['filebeat'], + 'agent.ephemeral_id': ['a0c8164b-3564-4e32-b0bf-f4db5a7ae566'], + 'agent.version': ['7.0.0'], tags: ['prod', 'web'], metadata: [ { key: 'env', value: 'prod' }, { key: 'stack', value: 'web' }, ], - host: { - hostname: 'packer-virtualbox-iso-1546820004', - name: 'demo-stack-client-01', - }, + 'host.hostname': ['packer-virtualbox-iso-1546820004'], + 'host.name': ['demo-stack-client-01'], }; - const fields = convertDocumentSourceToLogItemFields(doc); + const fields = convertESFieldsToLogItemFields(esFields); expect(fields).toEqual([ { field: 'agent.hostname', - value: 'demo-stack-client-01', + value: ['demo-stack-client-01'], }, { field: 'agent.id', - value: '7adef8b6-2ab7-45cd-a0d5-b3baad735f1b', + value: ['7adef8b6-2ab7-45cd-a0d5-b3baad735f1b'], }, { field: 'agent.type', - value: 'filebeat', + value: ['filebeat'], }, { field: 'agent.ephemeral_id', - value: 'a0c8164b-3564-4e32-b0bf-f4db5a7ae566', + value: ['a0c8164b-3564-4e32-b0bf-f4db5a7ae566'], }, { field: 'agent.version', - value: '7.0.0', + value: ['7.0.0'], }, { field: 'tags', - value: '["prod","web"]', + value: ['prod', 'web'], }, { field: 'metadata', - value: '[{"key":"env","value":"prod"},{"key":"stack","value":"web"}]', + value: ['{"key":"env","value":"prod"}', '{"key":"stack","value":"web"}'], }, { field: 'host.hostname', - value: 'packer-virtualbox-iso-1546820004', + value: ['packer-virtualbox-iso-1546820004'], }, { field: 'host.name', - value: 'demo-stack-client-01', + value: ['demo-stack-client-01'], }, ]); }); diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/convert_document_source_to_log_item_fields.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/convert_document_source_to_log_item_fields.ts index 7c8560d72ff97..a1d855bfdaa48 100644 --- a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/convert_document_source_to_log_item_fields.ts +++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/convert_document_source_to_log_item_fields.ts @@ -5,39 +5,21 @@ */ import stringify from 'json-stable-stringify'; -import { isArray, isPlainObject } from 'lodash'; - -import { JsonObject } from '../../../../common/typed_json'; import { LogEntriesItemField } from '../../../../common/http_api'; +import { JsonArray } from '../../../../common/typed_json'; -const isJsonObject = (subject: any): subject is JsonObject => { - return isPlainObject(subject); +const serializeValue = (value: JsonArray): string[] => { + return value.map((v) => { + if (typeof v === 'object' && v != null) { + return stringify(v); + } else { + return `${v}`; + } + }); }; -const serializeValue = (value: any): string => { - if (isArray(value) || isPlainObject(value)) { - return stringify(value); - } - return `${value}`; -}; export const convertESFieldsToLogItemFields = (fields: { - [field: string]: [value: unknown]; + [field: string]: JsonArray; }): LogEntriesItemField[] => { - return Object.keys(fields).map((field) => ({ field, value: serializeValue(fields[field][0]) })); -}; - -export const convertDocumentSourceToLogItemFields = ( - source: JsonObject, - path: string[] = [], - fields: LogEntriesItemField[] = [] -): LogEntriesItemField[] => { - return Object.keys(source).reduce((acc, key) => { - const value = source[key]; - const nextPath = [...path, key]; - if (isJsonObject(value)) { - return convertDocumentSourceToLogItemFields(value, nextPath, acc); - } - const field = { field: nextPath.join('.'), value: serializeValue(value) }; - return [...acc, field]; - }, fields); + return Object.keys(fields).map((field) => ({ field, value: serializeValue(fields[field]) })); }; diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts index e211f72b4e076..cc9d4c749c77d 100644 --- a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts +++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts @@ -7,7 +7,7 @@ import { sortBy } from 'lodash'; import { RequestHandlerContext } from 'src/core/server'; -import { JsonObject } from '../../../../common/typed_json'; +import { JsonArray, JsonObject } from '../../../../common/typed_json'; import { LogEntriesSummaryBucket, LogEntriesSummaryHighlightsBucket, @@ -163,8 +163,8 @@ export class InfraLogEntriesDomain { return { columnId: column.fieldColumn.id, field: column.fieldColumn.field, - value: doc.fields[column.fieldColumn.field], - highlights: doc.highlights[column.fieldColumn.field] || [], + value: doc.fields[column.fieldColumn.field] ?? [], + highlights: doc.highlights[column.fieldColumn.field] ?? [], }; } } @@ -252,8 +252,8 @@ export class InfraLogEntriesDomain { ): Promise { const document = await this.adapter.getLogItem(requestContext, id, sourceConfiguration); const defaultFields = [ - { field: '_index', value: document._index }, - { field: '_id', value: document._id }, + { field: '_index', value: [document._index] }, + { field: '_id', value: [document._id] }, ]; return { @@ -310,10 +310,10 @@ export class InfraLogEntriesDomain { } } -interface LogItemHit { +export interface LogItemHit { _index: string; _id: string; - fields: { [field: string]: [value: unknown] }; + fields: { [field: string]: [value: JsonArray] }; sort: [number, number]; } @@ -400,9 +400,9 @@ const createHighlightQueryDsl = (phrase: string, fields: string[]) => ({ const getContextFromDoc = (doc: LogEntryDocument): LogEntry['context'] => { // Get all context fields, then test for the presence and type of the ones that go together - const containerId = doc.fields['container.id']; - const hostName = doc.fields['host.name']; - const logFilePath = doc.fields['log.file.path']; + const containerId = doc.fields['container.id']?.[0]; + const hostName = doc.fields['host.name']?.[0]; + const logFilePath = doc.fields['log.file.path']?.[0]; if (typeof containerId === 'string') { return { 'container.id': containerId }; diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/message.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/message.ts index b8cadaa06f61b..f9775e127088a 100644 --- a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/message.ts +++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/message.ts @@ -4,11 +4,11 @@ * you may not use this file except in compliance with the Elastic License. */ -import stringify from 'json-stable-stringify'; - -import { InfraLogMessageSegment } from '../../../graphql/types'; +import { LogMessagePart } from '../../../../common/http_api/log_entries'; +import { JsonArray, JsonValue } from '../../../../common/typed_json'; import { LogMessageFormattingCondition, + LogMessageFormattingFieldValueConditionValue, LogMessageFormattingInstruction, LogMessageFormattingRule, } from './rule_types'; @@ -30,7 +30,7 @@ export function compileFormattingRules( ) ) ), - format(fields, highlights): InfraLogMessageSegment[] { + format(fields, highlights): LogMessagePart[] { for (const compiledRule of compiledRules) { if (compiledRule.fulfillsCondition(fields)) { return compiledRule.format(fields, highlights); @@ -59,16 +59,34 @@ const compileRule = (rule: LogMessageFormattingRule): CompiledLogMessageFormatti const compileCondition = ( condition: LogMessageFormattingCondition ): CompiledLogMessageFormattingCondition => - [compileExistsCondition, compileFieldValueCondition].reduce( - (compiledCondition, compile) => compile(condition) || compiledCondition, - catchAllCondition - ); + [ + compileAllCondition, + compileExistsCondition, + compileExistsPrefixCondition, + compileFieldValueCondition, + ].reduce((compiledCondition, compile) => compile(condition) || compiledCondition, falseCondition); -const catchAllCondition: CompiledLogMessageFormattingCondition = { +const falseCondition: CompiledLogMessageFormattingCondition = { conditionFields: [] as string[], fulfillsCondition: () => false, }; +const compileAllCondition = ( + condition: LogMessageFormattingCondition +): CompiledLogMessageFormattingCondition | null => { + if (!('all' in condition)) { + return null; + } + + const compiledConditions = condition.all.map(compileCondition); + + return { + conditionFields: compiledConditions.flatMap(({ conditionFields }) => conditionFields), + fulfillsCondition: (fields: Fields) => + compiledConditions.every(({ fulfillsCondition }) => fulfillsCondition(fields)), + }; +}; + const compileExistsCondition = (condition: LogMessageFormattingCondition) => 'exists' in condition ? { @@ -78,13 +96,24 @@ const compileExistsCondition = (condition: LogMessageFormattingCondition) => } : null; +const compileExistsPrefixCondition = (condition: LogMessageFormattingCondition) => + 'existsPrefix' in condition + ? { + conditionFields: condition.existsPrefix.map((prefix) => `${prefix}.*`), + fulfillsCondition: (fields: Fields) => + condition.existsPrefix.every((fieldNamePrefix) => + Object.keys(fields).some((field) => field.startsWith(`${fieldNamePrefix}.`)) + ), + } + : null; + const compileFieldValueCondition = (condition: LogMessageFormattingCondition) => 'values' in condition ? { conditionFields: Object.keys(condition.values), fulfillsCondition: (fields: Fields) => - Object.entries(condition.values).every( - ([fieldName, expectedValue]) => fields[fieldName] === expectedValue + Object.entries(condition.values).every(([fieldName, expectedValue]) => + equalsOrContains(fields[fieldName] ?? [], expectedValue) ), } : null; @@ -116,7 +145,11 @@ const compileFormattingInstructions = ( const compileFormattingInstruction = ( formattingInstruction: LogMessageFormattingInstruction ): CompiledLogMessageFormattingInstruction => - [compileFieldReferenceFormattingInstruction, compileConstantFormattingInstruction].reduce( + [ + compileFieldReferenceFormattingInstruction, + compileFieldsPrefixReferenceFormattingInstruction, + compileConstantFormattingInstruction, + ].reduce( (compiledFormattingInstruction, compile) => compile(formattingInstruction) || compiledFormattingInstruction, catchAllFormattingInstruction @@ -138,19 +171,44 @@ const compileFieldReferenceFormattingInstruction = ( ? { formattingFields: [formattingInstruction.field], format: (fields, highlights) => { - const value = fields[formattingInstruction.field]; - const highlightedValues = highlights[formattingInstruction.field]; + const value = fields[formattingInstruction.field] ?? []; + const highlightedValues = highlights[formattingInstruction.field] ?? []; return [ { field: formattingInstruction.field, - value: typeof value === 'object' ? stringify(value) : `${value}`, - highlights: highlightedValues || [], + value, + highlights: highlightedValues, }, ]; }, } : null; +const compileFieldsPrefixReferenceFormattingInstruction = ( + formattingInstruction: LogMessageFormattingInstruction +): CompiledLogMessageFormattingInstruction | null => + 'fieldsPrefix' in formattingInstruction + ? { + formattingFields: [`${formattingInstruction.fieldsPrefix}.*`], + format: (fields, highlights) => { + const matchingFields = Object.keys(fields).filter((field) => + field.startsWith(`${formattingInstruction.fieldsPrefix}.`) + ); + return matchingFields.flatMap((field) => { + const value = fields[field] ?? []; + const highlightedValues = highlights[field] ?? []; + return [ + { + field, + value, + highlights: highlightedValues, + }, + ]; + }); + }, + } + : null; + const compileConstantFormattingInstruction = ( formattingInstruction: LogMessageFormattingInstruction ): CompiledLogMessageFormattingInstruction | null => @@ -165,8 +223,21 @@ const compileConstantFormattingInstruction = ( } : null; +const equalsOrContains = ( + operand: JsonValue, + value: LogMessageFormattingFieldValueConditionValue +): boolean => { + if (Array.isArray(operand)) { + return operand.includes(value); + } else if (typeof operand === 'object' && operand !== null) { + return Object.values(operand).includes(value); + } else { + return operand === value; + } +}; + export interface Fields { - [fieldName: string]: string | number | object | boolean | null; + [fieldName: string]: JsonArray; } export interface Highlights { @@ -176,7 +247,7 @@ export interface Highlights { export interface CompiledLogMessageFormattingRule { requiredFields: string[]; fulfillsCondition(fields: Fields): boolean; - format(fields: Fields, highlights: Highlights): InfraLogMessageSegment[]; + format(fields: Fields, highlights: Highlights): LogMessagePart[]; } export interface CompiledLogMessageFormattingCondition { @@ -186,5 +257,5 @@ export interface CompiledLogMessageFormattingCondition { export interface CompiledLogMessageFormattingInstruction { formattingFields: string[]; - format(fields: Fields, highlights: Highlights): InfraLogMessageSegment[]; + format(fields: Fields, highlights: Highlights): LogMessagePart[]; } diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/rule_types.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/rule_types.ts index 6107fc362f8e3..4569f4b8e8a91 100644 --- a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/rule_types.ts +++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/rule_types.ts @@ -4,33 +4,52 @@ * you may not use this file except in compliance with the Elastic License. */ +import { JsonValue } from '../../../../common/typed_json'; + export interface LogMessageFormattingRule { when: LogMessageFormattingCondition; format: LogMessageFormattingInstruction[]; } export type LogMessageFormattingCondition = + | LogMessageFormattingAllCondition | LogMessageFormattingExistsCondition + | LogMessageFormattingExistsPrefixCondition | LogMessageFormattingFieldValueCondition; +export interface LogMessageFormattingAllCondition { + all: LogMessageFormattingCondition[]; +} + export interface LogMessageFormattingExistsCondition { exists: string[]; } +export interface LogMessageFormattingExistsPrefixCondition { + existsPrefix: string[]; +} + export interface LogMessageFormattingFieldValueCondition { values: { - [fieldName: string]: string | number | boolean | null; + [fieldName: string]: LogMessageFormattingFieldValueConditionValue; }; } +export type LogMessageFormattingFieldValueConditionValue = JsonValue; + export type LogMessageFormattingInstruction = | LogMessageFormattingFieldReference + | LogMessageFormattingFieldsPrefixReference | LogMessageFormattingConstant; export interface LogMessageFormattingFieldReference { field: string; } +export interface LogMessageFormattingFieldsPrefixReference { + fieldsPrefix: string; +} + export interface LogMessageFormattingConstant { constant: string; } diff --git a/x-pack/test/api_integration/apis/metrics_ui/log_item.ts b/x-pack/test/api_integration/apis/metrics_ui/log_item.ts index 6e0148f13277c..3bb7a9a76690d 100644 --- a/x-pack/test/api_integration/apis/metrics_ui/log_item.ts +++ b/x-pack/test/api_integration/apis/metrics_ui/log_item.ts @@ -44,107 +44,107 @@ export default function ({ getService }: FtrProviderContext) { expect(logItem.fields).to.eql([ { field: '@timestamp', - value: '2018-10-17T19:42:22.000Z', + value: ['2018-10-17T19:42:22.000Z'], }, { field: '_id', - value: 'yT2Mg2YBh-opCxJv8Vqj', + value: ['yT2Mg2YBh-opCxJv8Vqj'], }, { field: '_index', - value: 'filebeat-7.0.0-alpha1-2018.10.17', + value: ['filebeat-7.0.0-alpha1-2018.10.17'], }, { field: 'apache2.access.body_sent.bytes', - value: '1336', + value: ['1336'], }, { field: 'apache2.access.http_version', - value: '1.1', + value: ['1.1'], }, { field: 'apache2.access.method', - value: 'GET', + value: ['GET'], }, { field: 'apache2.access.referrer', - value: '-', + value: ['-'], }, { field: 'apache2.access.remote_ip', - value: '10.128.0.11', + value: ['10.128.0.11'], }, { field: 'apache2.access.response_code', - value: '200', + value: ['200'], }, { field: 'apache2.access.url', - value: '/a-fresh-start-will-put-you-on-your-way', + value: ['/a-fresh-start-will-put-you-on-your-way'], }, { field: 'apache2.access.user_agent.device', - value: 'Other', + value: ['Other'], }, { field: 'apache2.access.user_agent.name', - value: 'Other', + value: ['Other'], }, { field: 'apache2.access.user_agent.os', - value: 'Other', + value: ['Other'], }, { field: 'apache2.access.user_agent.os_name', - value: 'Other', + value: ['Other'], }, { field: 'apache2.access.user_name', - value: '-', + value: ['-'], }, { field: 'beat.hostname', - value: 'demo-stack-apache-01', + value: ['demo-stack-apache-01'], }, { field: 'beat.name', - value: 'demo-stack-apache-01', + value: ['demo-stack-apache-01'], }, { field: 'beat.version', - value: '7.0.0-alpha1', + value: ['7.0.0-alpha1'], }, { field: 'fileset.module', - value: 'apache2', + value: ['apache2'], }, { field: 'fileset.name', - value: 'access', + value: ['access'], }, { field: 'host.name', - value: 'demo-stack-apache-01', + value: ['demo-stack-apache-01'], }, { field: 'input.type', - value: 'log', + value: ['log'], }, { field: 'offset', - value: '5497614', + value: ['5497614'], }, { field: 'prospector.type', - value: 'log', + value: ['log'], }, { field: 'read_timestamp', - value: '2018-10-17T19:42:23.160Z', + value: ['2018-10-17T19:42:23.160Z'], }, { field: 'source', - value: '/var/log/apache2/access.log', + value: ['/var/log/apache2/access.log'], }, ]); });