diff --git a/.eslintrc.js b/.eslintrc.js index e66331594b4ae..3c173e5244009 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -552,29 +552,6 @@ module.exports = { }, }, - /** - * Graph overrides - */ - { - files: ['x-pack/legacy/plugins/graph/**/*.js'], - globals: { - angular: true, - $: true, - }, - rules: { - 'block-scoped-var': 'off', - camelcase: 'off', - eqeqeq: 'off', - 'guard-for-in': 'off', - 'new-cap': 'off', - 'no-loop-func': 'off', - 'no-redeclare': 'off', - 'no-shadow': 'off', - 'no-unused-vars': 'off', - 'one-var': 'off', - }, - }, - /** * ML overrides */ @@ -771,7 +748,7 @@ module.exports = { * Lens overrides */ { - files: ['x-pack/legacy/plugins/lens/**/*.ts', 'x-pack/legacy/plugins/lens/**/*.tsx'], + files: ['x-pack/legacy/plugins/lens/**/*.{ts,tsx}', 'x-pack/plugins/lens/**/*.{ts,tsx}'], rules: { '@typescript-eslint/no-explicit-any': 'error', }, @@ -885,8 +862,10 @@ module.exports = { * TSVB overrides */ { - files: ['src/legacy/core_plugins/metrics/**/*.js'], - excludedFiles: 'src/legacy/core_plugins/metrics/index.js', + files: [ + 'src/plugins/vis_type_timeseries/**/*.{js,ts,tsx}', + 'src/legacy/core_plugins/vis_type_timeseries/**/*.{js,ts,tsx}', + ], rules: { 'import/no-default-export': 'error', }, diff --git a/docs/apm/images/service-maps-java.png b/docs/apm/images/service-maps-java.png new file mode 100644 index 0000000000000..e1a42f4c76e12 Binary files /dev/null and b/docs/apm/images/service-maps-java.png differ diff --git a/docs/apm/images/service-maps.png b/docs/apm/images/service-maps.png new file mode 100644 index 0000000000000..454ae9bb720fb Binary files /dev/null and b/docs/apm/images/service-maps.png differ diff --git a/docs/apm/service-maps.asciidoc b/docs/apm/service-maps.asciidoc new file mode 100644 index 0000000000000..e0d84f33b4dcb --- /dev/null +++ b/docs/apm/service-maps.asciidoc @@ -0,0 +1,48 @@ +[[service-maps]] +=== Service maps + +beta::[] + +A service map is a real-time diagram of the interactions occurring in your application’s architecture. +It allows you to easily visualize data flow and high-level statistics, like average transaction duration, +requests per minute, errors per minute, and metrics, allowing you to quickly assess the status of your services. + +Our beta offering creates two types of service maps: + +* Global: All services and connections are shown. +* Service-specific: Selecting a specific service will highlight it's connections. + +[role="screenshot"] +image::apm/images/service-maps.png[Example view of service maps in the APM app in Kibana] + +[float] +[[visualize-your-architecture]] +=== Visualize your architecture + +Select the **Service Map** tab to get started. +By default, all services and connections are shown. +Whether your onboarding a new engineer, or just trying to grasp the big picture, +click around, zoom in and out, and begin to visualize how your services are connected. + +If there's a specific service that interests you, select that service to highlight its connections. +Clicking **Focus map** will refocus the map on that specific service and lock the connection highlighting. +From here, select **Service Details**, or click on the **Transaction** tab to jump to the Transaction overview. +You can also use the tabs at the top of the page to easily jump to the **Errors** or **Metrics** overview. + +While it's not possible to query in service maps, it is possible to filter by environment. +This can be useful if you have two or more services, in separate environments, but with the same name. +Use the environment drop down to only see the data you're interested in, like `dev` or `production`. + +[role="screenshot"] +image::apm/images/service-maps-java.png[Example view of service maps with Java highlighted in the APM app in Kibana] + +[float] +[[service-maps-legend]] +=== Legend + +Nodes appear on the map in one of two shapes: + +* **Circle**: Instrumented services. Interior icons are based on the language of the agent used. +* **Diamond**: Databases, external, and messaging. Interior icons represent the generic type, +with specific icons for known entities, like Elasticsearch. +Type and subtype are based on `span.type`, and `span.subtype`. diff --git a/docs/apm/using-the-apm-ui.asciidoc b/docs/apm/using-the-apm-ui.asciidoc index 1361dc046e3b1..b1b7ed7307986 100644 --- a/docs/apm/using-the-apm-ui.asciidoc +++ b/docs/apm/using-the-apm-ui.asciidoc @@ -31,6 +31,8 @@ include::transactions.asciidoc[] include::spans.asciidoc[] +include::service-maps.asciidoc[] + include::errors.asciidoc[] include::metrics.asciidoc[] diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.search.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.search.md index afb6ea88f9fad..78ac05b9fd386 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.search.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.search.md @@ -19,7 +19,7 @@ search: { intervalOptions: ({ display: string; val: string; - enabled(agg: import("./search/aggs/buckets/_bucket_agg_type").IBucketAggConfig): boolean | "" | undefined; + enabled(agg: import("./search/aggs/buckets/bucket_agg_type").IBucketAggConfig): boolean | "" | undefined; } | { display: string; val: string; diff --git a/src/legacy/ui/public/new_platform/new_platform.karma_mock.js b/src/legacy/ui/public/new_platform/new_platform.karma_mock.js index 25647e4a08897..f70ef069dd134 100644 --- a/src/legacy/ui/public/new_platform/new_platform.karma_mock.js +++ b/src/legacy/ui/public/new_platform/new_platform.karma_mock.js @@ -88,6 +88,9 @@ const mockCoreStart = { get: sinon.fake.returns(''), }, }, + notifications: { + toasts: {}, + }, i18n: {}, overlays: {}, savedObjects: { @@ -164,8 +167,11 @@ const mockAggTypesRegistry = () => { const registrySetup = registry.setup(); const aggTypes = getAggTypes({ uiSettings: mockCoreSetup.uiSettings, - notifications: mockCoreStart.notifications, query: querySetup, + getInternalStartServices: () => ({ + fieldFormats: getFieldFormatsRegistry(mockCoreStart), + notifications: mockCoreStart.notifications, + }), }); aggTypes.buckets.forEach(type => registrySetup.registerBucket(type)); aggTypes.metrics.forEach(type => registrySetup.registerMetric(type)); diff --git a/src/plugins/data/common/field_formats/mocks.ts b/src/plugins/data/common/field_formats/mocks.ts index bc38374e147cf..394d4c383032f 100644 --- a/src/plugins/data/common/field_formats/mocks.ts +++ b/src/plugins/data/common/field_formats/mocks.ts @@ -17,23 +17,14 @@ * under the License. */ -import { FieldFormat, IFieldFormatsRegistry } from '.'; - -const fieldFormatMock = ({ - convert: jest.fn(), - getConverterFor: jest.fn(), - getParamDefaults: jest.fn(), - param: jest.fn(), - params: jest.fn(), - toJSON: jest.fn(), - type: jest.fn(), - setupContentType: jest.fn(), -} as unknown) as FieldFormat; +import { IFieldFormatsRegistry } from '.'; export const fieldFormatsMock: IFieldFormatsRegistry = { getByFieldType: jest.fn(), getDefaultConfig: jest.fn(), - getDefaultInstance: jest.fn().mockImplementation(() => fieldFormatMock) as any, + getDefaultInstance: jest.fn().mockImplementation(() => ({ + getConverterFor: jest.fn().mockImplementation(() => (t: string) => t), + })) as any, getDefaultInstanceCacheResolver: jest.fn(), getDefaultInstancePlain: jest.fn(), getDefaultType: jest.fn(), diff --git a/src/plugins/data/public/field_formats/mocks.ts b/src/plugins/data/public/field_formats/mocks.ts new file mode 100644 index 0000000000000..ec1233a085bce --- /dev/null +++ b/src/plugins/data/public/field_formats/mocks.ts @@ -0,0 +1,41 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { FieldFormatsStart, FieldFormatsSetup, FieldFormatsService } from '.'; +import { fieldFormatsMock } from '../../common/field_formats/mocks'; + +type FieldFormatsServiceClientContract = PublicMethodsOf; + +const createSetupContractMock = () => fieldFormatsMock as FieldFormatsSetup; +const createStartContractMock = () => fieldFormatsMock as FieldFormatsStart; + +const createMock = () => { + const mocked: jest.Mocked = { + setup: jest.fn().mockReturnValue(createSetupContractMock()), + start: jest.fn().mockReturnValue(createStartContractMock()), + }; + + return mocked; +}; + +export const fieldFormatsServiceMock = { + create: createMock, + createSetupContract: createSetupContractMock, + createStartContract: createStartContractMock, +}; diff --git a/src/plugins/data/public/index_patterns/index_patterns/index_patterns_api_client.ts b/src/plugins/data/public/index_patterns/index_patterns/index_patterns_api_client.ts index 4d4e8d8827b48..0007d1780c25b 100644 --- a/src/plugins/data/public/index_patterns/index_patterns/index_patterns_api_client.ts +++ b/src/plugins/data/public/index_patterns/index_patterns/index_patterns_api_client.ts @@ -45,7 +45,7 @@ export class IndexPatternsApiClient { query, }) .catch((resp: any) => { - if (resp.body.statusCode === 404 && resp.body.statuscode === 'no_matching_indices') { + if (resp.body.statusCode === 404 && resp.body.attributes?.code === 'no_matching_indices') { throw new IndexPatternMissingIndices(resp.body.message); } diff --git a/src/plugins/data/public/mocks.ts b/src/plugins/data/public/mocks.ts index e3fc0e97af09b..ea1c27550867e 100644 --- a/src/plugins/data/public/mocks.ts +++ b/src/plugins/data/public/mocks.ts @@ -17,8 +17,8 @@ * under the License. */ -import { Plugin, DataPublicPluginSetup, DataPublicPluginStart, IndexPatternsContract } from '.'; -import { fieldFormatsMock } from '../common/field_formats/mocks'; +import { Plugin, IndexPatternsContract } from '.'; +import { fieldFormatsServiceMock } from './field_formats/mocks'; import { searchSetupMock, searchStartMock } from './search/mocks'; import { queryServiceMock } from './query/mocks'; @@ -36,7 +36,7 @@ const createSetupContract = (): Setup => { return { autocomplete: autocompleteMock, search: searchSetupMock, - fieldFormats: fieldFormatsMock as DataPublicPluginSetup['fieldFormats'], + fieldFormats: fieldFormatsServiceMock.createSetupContract(), query: querySetupMock, }; }; @@ -49,7 +49,7 @@ const createStartContract = (): Start => { }, autocomplete: autocompleteMock, search: searchStartMock, - fieldFormats: fieldFormatsMock as DataPublicPluginStart['fieldFormats'], + fieldFormats: fieldFormatsServiceMock.createStartContract(), query: queryStartMock, ui: { IndexPatternSelect: jest.fn(), diff --git a/src/plugins/data/public/plugin.ts b/src/plugins/data/public/plugin.ts index 26587470adfd9..15067077afc43 100644 --- a/src/plugins/data/public/plugin.ts +++ b/src/plugins/data/public/plugin.ts @@ -30,6 +30,7 @@ import { DataPublicPluginStart, DataSetupDependencies, DataStartDependencies, + GetInternalStartServicesFn, } from './types'; import { AutocompleteService } from './autocomplete'; import { SearchService } from './search/search_service'; @@ -47,6 +48,8 @@ import { setQueryService, setSearchService, setUiSettings, + getFieldFormats, + getNotifications, } from './services'; import { createSearchBar } from './ui/search_bar/create_search_bar'; import { esaggs } from './search/expressions'; @@ -100,6 +103,11 @@ export class DataPublicPlugin implements Plugin ({ + fieldFormats: getFieldFormats(), + notifications: getNotifications(), + }); + const queryService = this.queryService.setup({ uiSettings: core.uiSettings, storage: this.storage, @@ -122,6 +130,7 @@ export class DataPublicPlugin implements Plugin { +export interface IDataPluginServices extends Partial { // (undocumented) appName: string; // (undocumented) data: DataPublicPluginStart; // (undocumented) - http: CoreStart_2['http']; + http: CoreStart['http']; // (undocumented) - notifications: CoreStart_2['notifications']; + notifications: CoreStart['notifications']; // (undocumented) - savedObjects: CoreStart_2['savedObjects']; + savedObjects: CoreStart['savedObjects']; // (undocumented) storage: IStorageWrapper; // (undocumented) - uiSettings: CoreStart_2['uiSettings']; + uiSettings: CoreStart['uiSettings']; } // Warning: (ae-missing-release-tag) "IEsSearchRequest" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) @@ -1094,7 +1094,7 @@ export type ISearch = // @public (undocumented) export interface ISearchContext { // (undocumented) - core: CoreStart; + core: CoreStart_2; // (undocumented) getSearchStrategy: (name: T) => TSearchStrategyProvider; } @@ -1307,7 +1307,7 @@ export class Plugin implements Plugin_2 { let indexPattern: IndexPattern; @@ -400,13 +398,6 @@ describe('AggConfig', () => { describe('#fieldFormatter - custom getFormat handler', () => { it('returns formatter from getFormat handler', () => { - setFieldFormats({ - ...dataPluginMock.createStartContract().fieldFormats, - getDefaultInstance: jest.fn().mockImplementation(() => ({ - getConverterFor: jest.fn().mockImplementation(() => (t: string) => t), - })) as any, - }); - const ac = new AggConfigs(indexPattern, [], { typesRegistry }); const configStates = { enabled: true, @@ -429,12 +420,6 @@ describe('AggConfig', () => { let aggConfig: AggConfig; beforeEach(() => { - setFieldFormats({ - ...dataPluginMock.createStartContract().fieldFormats, - getDefaultInstance: jest.fn().mockImplementation(() => ({ - getConverterFor: (t?: string) => t || identity, - })) as any, - }); indexPattern.fields.getByName = name => ({ format: { diff --git a/src/plugins/data/public/search/aggs/agg_params.test.ts b/src/plugins/data/public/search/aggs/agg_params.test.ts index b08fcf309e9ed..784be803e2644 100644 --- a/src/plugins/data/public/search/aggs/agg_params.test.ts +++ b/src/plugins/data/public/search/aggs/agg_params.test.ts @@ -22,12 +22,22 @@ import { BaseParamType } from './param_types/base'; import { FieldParamType } from './param_types/field'; import { OptionedParamType } from './param_types/optioned'; import { AggParamType } from '../aggs/param_types/agg'; +import { fieldFormatsServiceMock } from '../../field_formats/mocks'; +import { notificationServiceMock } from '../../../../../../src/core/public/mocks'; +import { AggTypeDependencies } from './agg_type'; describe('AggParams class', () => { + const aggTypesDependencies: AggTypeDependencies = { + getInternalStartServices: () => ({ + fieldFormats: fieldFormatsServiceMock.createStartContract(), + notifications: notificationServiceMock.createStartContract(), + }), + }; + describe('constructor args', () => { it('accepts an array of param defs', () => { const params = [{ name: 'one' }, { name: 'two' }] as AggParamType[]; - const aggParams = initParams(params); + const aggParams = initParams(params, aggTypesDependencies); expect(aggParams).toHaveLength(params.length); expect(Array.isArray(aggParams)).toBeTruthy(); @@ -37,7 +47,7 @@ describe('AggParams class', () => { describe('AggParam creation', () => { it('Uses the FieldParamType class for params with the name "field"', () => { const params = [{ name: 'field', type: 'field' }] as AggParamType[]; - const aggParams = initParams(params); + const aggParams = initParams(params, aggTypesDependencies); expect(aggParams).toHaveLength(params.length); expect(aggParams[0] instanceof FieldParamType).toBeTruthy(); @@ -50,7 +60,7 @@ describe('AggParams class', () => { type: 'optioned', }, ] as AggParamType[]; - const aggParams = initParams(params); + const aggParams = initParams(params, aggTypesDependencies); expect(aggParams).toHaveLength(params.length); expect(aggParams[0] instanceof OptionedParamType).toBeTruthy(); @@ -72,7 +82,7 @@ describe('AggParams class', () => { }, ] as AggParamType[]; - const aggParams = initParams(params); + const aggParams = initParams(params, aggTypesDependencies); expect(aggParams).toHaveLength(params.length); diff --git a/src/plugins/data/public/search/aggs/agg_params.ts b/src/plugins/data/public/search/aggs/agg_params.ts index 551cb81529a0a..e7b2f72bae656 100644 --- a/src/plugins/data/public/search/aggs/agg_params.ts +++ b/src/plugins/data/public/search/aggs/agg_params.ts @@ -26,6 +26,7 @@ import { BaseParamType } from './param_types/base'; import { AggConfig } from './agg_config'; import { IAggConfigs } from './agg_configs'; +import { AggTypeDependencies } from './agg_type'; const paramTypeMap = { field: FieldParamType, @@ -45,12 +46,13 @@ export interface AggParamOption { } export const initParams = ( - params: TAggParam[] + params: TAggParam[], + { getInternalStartServices }: AggTypeDependencies ): TAggParam[] => params.map((config: TAggParam) => { const Class = paramTypeMap[config.type] || paramTypeMap._default; - return new Class(config); + return new Class(config, { getInternalStartServices }); }) as TAggParam[]; /** diff --git a/src/plugins/data/public/search/aggs/agg_type.test.ts b/src/plugins/data/public/search/aggs/agg_type.test.ts index 3fb03dc31e2b2..0c9e110c34ae6 100644 --- a/src/plugins/data/public/search/aggs/agg_type.test.ts +++ b/src/plugins/data/public/search/aggs/agg_type.test.ts @@ -17,40 +17,49 @@ * under the License. */ -import { AggType, AggTypeConfig } from './agg_type'; +import { AggType, AggTypeConfig, AggTypeDependencies } from './agg_type'; import { IAggConfig } from './agg_config'; -import { mockDataServices } from './test_helpers'; -import { dataPluginMock } from '../../../public/mocks'; -import { setFieldFormats } from '../../../public/services'; +import { fieldFormatsServiceMock } from '../../field_formats/mocks'; +import { notificationServiceMock } from '../../../../../../src/core/public/mocks'; describe('AggType Class', () => { + let dependencies: AggTypeDependencies; + beforeEach(() => { - mockDataServices(); + dependencies = { + getInternalStartServices: () => ({ + fieldFormats: { + ...fieldFormatsServiceMock.createStartContract(), + getDefaultInstance: jest.fn(() => 'default') as any, + }, + notifications: notificationServiceMock.createStartContract(), + }), + }; }); describe('constructor', () => { - it("requires a valid config object as it's first param", () => { + test("requires a valid config object as it's first param", () => { expect(() => { const aggConfig: AggTypeConfig = (undefined as unknown) as AggTypeConfig; - new AggType(aggConfig); + new AggType(aggConfig, dependencies); }).toThrowError(); }); describe('application of config properties', () => { - it('assigns the config value to itself', () => { + test('assigns the config value to itself', () => { const config: AggTypeConfig = { name: 'name', title: 'title', }; - const aggType = new AggType(config); + const aggType = new AggType(config, dependencies); expect(aggType.name).toBe('name'); expect(aggType.title).toBe('title'); }); describe('makeLabel', () => { - it('makes a function when the makeLabel config is not specified', () => { + test('makes a function when the makeLabel config is not specified', () => { const makeLabel = () => 'label'; const aggConfig = {} as IAggConfig; const config: AggTypeConfig = { @@ -59,7 +68,7 @@ describe('AggType Class', () => { makeLabel, }; - const aggType = new AggType(config); + const aggType = new AggType(config, dependencies); expect(aggType.makeLabel).toBe(makeLabel); expect(aggType.makeLabel(aggConfig)).toBe('label'); @@ -67,26 +76,32 @@ describe('AggType Class', () => { }); describe('getResponseAggs/getRequestAggs', () => { - it('copies the value', () => { + test('copies the value', () => { const testConfig = (aggConfig: IAggConfig) => [aggConfig]; - const aggType = new AggType({ - name: 'name', - title: 'title', - getResponseAggs: testConfig, - getRequestAggs: testConfig, - }); + const aggType = new AggType( + { + name: 'name', + title: 'title', + getResponseAggs: testConfig, + getRequestAggs: testConfig, + }, + dependencies + ); expect(aggType.getResponseAggs).toBe(testConfig); expect(aggType.getResponseAggs).toBe(testConfig); }); - it('defaults to noop', () => { + test('defaults to noop', () => { const aggConfig = {} as IAggConfig; - const aggType = new AggType({ - name: 'name', - title: 'title', - }); + const aggType = new AggType( + { + name: 'name', + title: 'title', + }, + dependencies + ); const responseAggs = aggType.getRequestAggs(aggConfig); expect(responseAggs).toBe(undefined); @@ -94,11 +109,14 @@ describe('AggType Class', () => { }); describe('params', () => { - it('defaults to AggParams object with JSON param', () => { - const aggType = new AggType({ - name: 'smart agg', - title: 'title', - }); + test('defaults to AggParams object with JSON param', () => { + const aggType = new AggType( + { + name: 'smart agg', + title: 'title', + }, + dependencies + ); expect(Array.isArray(aggType.params)).toBeTruthy(); expect(aggType.params.length).toBe(2); @@ -106,26 +124,32 @@ describe('AggType Class', () => { expect(aggType.params[1].name).toBe('customLabel'); }); - it('can disable customLabel', () => { - const aggType = new AggType({ - name: 'smart agg', - title: 'title', - customLabels: false, - }); + test('can disable customLabel', () => { + const aggType = new AggType( + { + name: 'smart agg', + title: 'title', + customLabels: false, + }, + dependencies + ); expect(aggType.params.length).toBe(1); expect(aggType.params[0].name).toBe('json'); }); - it('passes the params arg directly to the AggParams constructor', () => { + test('passes the params arg directly to the AggParams constructor', () => { const params = [{ name: 'one' }, { name: 'two' }]; const paramLength = params.length + 2; // json and custom label are always appended - const aggType = new AggType({ - name: 'bucketeer', - title: 'title', - params, - }); + const aggType = new AggType( + { + name: 'bucketeer', + title: 'title', + params, + }, + dependencies + ); expect(Array.isArray(aggType.params)).toBeTruthy(); expect(aggType.params.length).toBe(paramLength); @@ -143,11 +167,14 @@ describe('AggType Class', () => { } as unknown) as IAggConfig; }); - it('returns the formatter for the aggConfig', () => { - const aggType = new AggType({ - name: 'name', - title: 'title', - }); + test('returns the formatter for the aggConfig', () => { + const aggType = new AggType( + { + name: 'name', + title: 'title', + }, + dependencies + ); field = { format: 'format', @@ -156,16 +183,14 @@ describe('AggType Class', () => { expect(aggType.getFormat(aggConfig)).toBe('format'); }); - it('returns default formatter', () => { - setFieldFormats({ - ...dataPluginMock.createStartContract().fieldFormats, - getDefaultInstance: jest.fn(() => 'default') as any, - }); - - const aggType = new AggType({ - name: 'name', - title: 'title', - }); + test('returns default formatter', () => { + const aggType = new AggType( + { + name: 'name', + title: 'title', + }, + dependencies + ); field = undefined; diff --git a/src/plugins/data/public/search/aggs/agg_type.ts b/src/plugins/data/public/search/aggs/agg_type.ts index a63d01e196612..70c116d560c6f 100644 --- a/src/plugins/data/public/search/aggs/agg_type.ts +++ b/src/plugins/data/public/search/aggs/agg_type.ts @@ -28,7 +28,7 @@ import { BaseParamType } from './param_types/base'; import { AggParamType } from './param_types/agg'; import { KBN_FIELD_TYPES, IFieldFormat } from '../../../common'; import { ISearchSource } from '../search_source'; -import { getFieldFormats } from '../../../public/services'; +import { GetInternalStartServicesFn } from '../../types'; export interface AggTypeConfig< TAggConfig extends AggConfig = AggConfig, @@ -60,16 +60,13 @@ export interface AggTypeConfig< getKey?: (bucket: any, key: any, agg: TAggConfig) => any; } -const getFormat = (agg: AggConfig) => { - const field = agg.getField(); - const fieldFormatsService = getFieldFormats(); - - return field ? field.format : fieldFormatsService.getDefaultInstance(KBN_FIELD_TYPES.STRING); -}; - // TODO need to make a more explicit interface for this export type IAggType = AggType; +export interface AggTypeDependencies { + getInternalStartServices: GetInternalStartServicesFn; +} + export class AggType< TAggConfig extends AggConfig = AggConfig, TParam extends AggParamType = AggParamType @@ -215,7 +212,10 @@ export class AggType< * @private * @param {object} config - used to set the properties of the AggType */ - constructor(config: AggTypeConfig) { + constructor( + config: AggTypeConfig, + { getInternalStartServices }: AggTypeDependencies + ) { this.name = config.name; this.type = config.type || 'metrics'; this.dslName = config.dslName || config.name; @@ -251,14 +251,22 @@ export class AggType< }); } - this.params = initParams(params); + this.params = initParams(params, { getInternalStartServices }); } this.getRequestAggs = config.getRequestAggs || noop; this.getResponseAggs = config.getResponseAggs || (() => {}); this.decorateAggConfig = config.decorateAggConfig || (() => ({})); this.postFlightRequest = config.postFlightRequest || identity; - this.getFormat = config.getFormat || getFormat; + + this.getFormat = + config.getFormat || + ((agg: TAggConfig) => { + const field = agg.getField(); + const { fieldFormats } = getInternalStartServices(); + + return field ? field.format : fieldFormats.getDefaultInstance(KBN_FIELD_TYPES.STRING); + }); this.getValue = config.getValue || ((agg: TAggConfig, bucket: any) => {}); } } diff --git a/src/plugins/data/public/search/aggs/agg_types.ts b/src/plugins/data/public/search/aggs/agg_types.ts index 556f6b0c93c41..4b154c338d48c 100644 --- a/src/plugins/data/public/search/aggs/agg_types.ts +++ b/src/plugins/data/public/search/aggs/agg_types.ts @@ -17,83 +17,89 @@ * under the License. */ -import { IUiSettingsClient, NotificationsSetup } from 'src/core/public'; +import { IUiSettingsClient } from 'src/core/public'; import { QuerySetup } from '../../query/query_service'; -import { countMetricAgg } from './metrics/count'; -import { avgMetricAgg } from './metrics/avg'; -import { sumMetricAgg } from './metrics/sum'; -import { medianMetricAgg } from './metrics/median'; -import { minMetricAgg } from './metrics/min'; -import { maxMetricAgg } from './metrics/max'; -import { topHitMetricAgg } from './metrics/top_hit'; -import { stdDeviationMetricAgg } from './metrics/std_deviation'; -import { cardinalityMetricAgg } from './metrics/cardinality'; -import { percentilesMetricAgg } from './metrics/percentiles'; -import { geoBoundsMetricAgg } from './metrics/geo_bounds'; -import { geoCentroidMetricAgg } from './metrics/geo_centroid'; -import { percentileRanksMetricAgg } from './metrics/percentile_ranks'; -import { derivativeMetricAgg } from './metrics/derivative'; -import { cumulativeSumMetricAgg } from './metrics/cumulative_sum'; -import { movingAvgMetricAgg } from './metrics/moving_avg'; -import { serialDiffMetricAgg } from './metrics/serial_diff'; +import { getCountMetricAgg } from './metrics/count'; +import { getAvgMetricAgg } from './metrics/avg'; +import { getSumMetricAgg } from './metrics/sum'; +import { getMedianMetricAgg } from './metrics/median'; +import { getMinMetricAgg } from './metrics/min'; +import { getMaxMetricAgg } from './metrics/max'; +import { getTopHitMetricAgg } from './metrics/top_hit'; +import { getStdDeviationMetricAgg } from './metrics/std_deviation'; +import { getCardinalityMetricAgg } from './metrics/cardinality'; +import { getPercentilesMetricAgg } from './metrics/percentiles'; +import { getGeoBoundsMetricAgg } from './metrics/geo_bounds'; +import { getGeoCentroidMetricAgg } from './metrics/geo_centroid'; +import { getPercentileRanksMetricAgg } from './metrics/percentile_ranks'; +import { getDerivativeMetricAgg } from './metrics/derivative'; +import { getCumulativeSumMetricAgg } from './metrics/cumulative_sum'; +import { getMovingAvgMetricAgg } from './metrics/moving_avg'; +import { getSerialDiffMetricAgg } from './metrics/serial_diff'; import { getDateHistogramBucketAgg } from './buckets/date_histogram'; import { getHistogramBucketAgg } from './buckets/histogram'; -import { rangeBucketAgg } from './buckets/range'; +import { getRangeBucketAgg } from './buckets/range'; import { getDateRangeBucketAgg } from './buckets/date_range'; -import { ipRangeBucketAgg } from './buckets/ip_range'; -import { termsBucketAgg } from './buckets/terms'; -import { filterBucketAgg } from './buckets/filter'; +import { getIpRangeBucketAgg } from './buckets/ip_range'; +import { getTermsBucketAgg } from './buckets/terms'; +import { getFilterBucketAgg } from './buckets/filter'; import { getFiltersBucketAgg } from './buckets/filters'; -import { significantTermsBucketAgg } from './buckets/significant_terms'; -import { geoHashBucketAgg } from './buckets/geo_hash'; -import { geoTileBucketAgg } from './buckets/geo_tile'; -import { bucketSumMetricAgg } from './metrics/bucket_sum'; -import { bucketAvgMetricAgg } from './metrics/bucket_avg'; -import { bucketMinMetricAgg } from './metrics/bucket_min'; -import { bucketMaxMetricAgg } from './metrics/bucket_max'; +import { getSignificantTermsBucketAgg } from './buckets/significant_terms'; +import { getGeoHashBucketAgg } from './buckets/geo_hash'; +import { getGeoTitleBucketAgg } from './buckets/geo_tile'; +import { getBucketSumMetricAgg } from './metrics/bucket_sum'; +import { getBucketAvgMetricAgg } from './metrics/bucket_avg'; +import { getBucketMinMetricAgg } from './metrics/bucket_min'; +import { getBucketMaxMetricAgg } from './metrics/bucket_max'; + +import { GetInternalStartServicesFn } from '../../types'; export interface AggTypesDependencies { - notifications: NotificationsSetup; uiSettings: IUiSettingsClient; query: QuerySetup; + getInternalStartServices: GetInternalStartServicesFn; } -export const getAggTypes = ({ notifications, uiSettings, query }: AggTypesDependencies) => ({ +export const getAggTypes = ({ + uiSettings, + query, + getInternalStartServices, +}: AggTypesDependencies) => ({ metrics: [ - countMetricAgg, - avgMetricAgg, - sumMetricAgg, - medianMetricAgg, - minMetricAgg, - maxMetricAgg, - stdDeviationMetricAgg, - cardinalityMetricAgg, - percentilesMetricAgg, - percentileRanksMetricAgg, - topHitMetricAgg, - derivativeMetricAgg, - cumulativeSumMetricAgg, - movingAvgMetricAgg, - serialDiffMetricAgg, - bucketAvgMetricAgg, - bucketSumMetricAgg, - bucketMinMetricAgg, - bucketMaxMetricAgg, - geoBoundsMetricAgg, - geoCentroidMetricAgg, + getCountMetricAgg({ getInternalStartServices }), + getAvgMetricAgg({ getInternalStartServices }), + getSumMetricAgg({ getInternalStartServices }), + getMedianMetricAgg({ getInternalStartServices }), + getMinMetricAgg({ getInternalStartServices }), + getMaxMetricAgg({ getInternalStartServices }), + getStdDeviationMetricAgg({ getInternalStartServices }), + getCardinalityMetricAgg({ getInternalStartServices }), + getPercentilesMetricAgg({ getInternalStartServices }), + getPercentileRanksMetricAgg({ getInternalStartServices }), + getTopHitMetricAgg({ getInternalStartServices }), + getDerivativeMetricAgg({ getInternalStartServices }), + getCumulativeSumMetricAgg({ getInternalStartServices }), + getMovingAvgMetricAgg({ getInternalStartServices }), + getSerialDiffMetricAgg({ getInternalStartServices }), + getBucketAvgMetricAgg({ getInternalStartServices }), + getBucketSumMetricAgg({ getInternalStartServices }), + getBucketMinMetricAgg({ getInternalStartServices }), + getBucketMaxMetricAgg({ getInternalStartServices }), + getGeoBoundsMetricAgg({ getInternalStartServices }), + getGeoCentroidMetricAgg({ getInternalStartServices }), ], buckets: [ - getDateHistogramBucketAgg({ uiSettings, query }), - getHistogramBucketAgg({ uiSettings, notifications }), - rangeBucketAgg, - getDateRangeBucketAgg({ uiSettings }), - ipRangeBucketAgg, - termsBucketAgg, - filterBucketAgg, - getFiltersBucketAgg({ uiSettings }), - significantTermsBucketAgg, - geoHashBucketAgg, - geoTileBucketAgg, + getDateHistogramBucketAgg({ uiSettings, query, getInternalStartServices }), + getHistogramBucketAgg({ uiSettings, getInternalStartServices }), + getRangeBucketAgg({ getInternalStartServices }), + getDateRangeBucketAgg({ uiSettings, getInternalStartServices }), + getIpRangeBucketAgg({ getInternalStartServices }), + getTermsBucketAgg({ getInternalStartServices }), + getFilterBucketAgg({ getInternalStartServices }), + getFiltersBucketAgg({ uiSettings, getInternalStartServices }), + getSignificantTermsBucketAgg({ getInternalStartServices }), + getGeoHashBucketAgg({ getInternalStartServices }), + getGeoTitleBucketAgg({ getInternalStartServices }), ], }); diff --git a/src/plugins/data/public/search/aggs/agg_types_registry.test.ts b/src/plugins/data/public/search/aggs/agg_types_registry.test.ts index 405f83e237de8..58d1a07d965e2 100644 --- a/src/plugins/data/public/search/aggs/agg_types_registry.test.ts +++ b/src/plugins/data/public/search/aggs/agg_types_registry.test.ts @@ -22,7 +22,7 @@ import { AggTypesRegistrySetup, AggTypesRegistryStart, } from './agg_types_registry'; -import { BucketAggType } from './buckets/_bucket_agg_type'; +import { BucketAggType } from './buckets/bucket_agg_type'; import { MetricAggType } from './metrics/metric_agg_type'; const bucketType = { name: 'terms', type: 'bucket' } as BucketAggType; diff --git a/src/plugins/data/public/search/aggs/agg_types_registry.ts b/src/plugins/data/public/search/aggs/agg_types_registry.ts index 8a8746106ae58..5a0c58120d810 100644 --- a/src/plugins/data/public/search/aggs/agg_types_registry.ts +++ b/src/plugins/data/public/search/aggs/agg_types_registry.ts @@ -17,7 +17,7 @@ * under the License. */ -import { BucketAggType } from './buckets/_bucket_agg_type'; +import { BucketAggType } from './buckets/bucket_agg_type'; import { MetricAggType } from './metrics/metric_agg_type'; export type AggTypesRegistrySetup = ReturnType; diff --git a/src/plugins/data/public/search/aggs/buckets/_interval_options.ts b/src/plugins/data/public/search/aggs/buckets/_interval_options.ts index 393d3b745250f..1c4c04c40a5c1 100644 --- a/src/plugins/data/public/search/aggs/buckets/_interval_options.ts +++ b/src/plugins/data/public/search/aggs/buckets/_interval_options.ts @@ -18,7 +18,7 @@ */ import { i18n } from '@kbn/i18n'; -import { IBucketAggConfig } from './_bucket_agg_type'; +import { IBucketAggConfig } from './bucket_agg_type'; export const intervalOptions = [ { diff --git a/src/plugins/data/public/search/aggs/buckets/_terms_other_bucket_helper.test.ts b/src/plugins/data/public/search/aggs/buckets/_terms_other_bucket_helper.test.ts index 9e4b93035384f..c664325a168b1 100644 --- a/src/plugins/data/public/search/aggs/buckets/_terms_other_bucket_helper.test.ts +++ b/src/plugins/data/public/search/aggs/buckets/_terms_other_bucket_helper.test.ts @@ -24,8 +24,8 @@ import { } from './_terms_other_bucket_helper'; import { AggConfigs, CreateAggConfigParams } from '../agg_configs'; import { BUCKET_TYPES } from './bucket_agg_types'; -import { IBucketAggConfig } from './_bucket_agg_type'; -import { mockDataServices, mockAggTypesRegistry } from '../test_helpers'; +import { IBucketAggConfig } from './bucket_agg_type'; +import { mockAggTypesRegistry } from '../test_helpers'; const indexPattern = { id: '1234', @@ -223,10 +223,6 @@ describe('Terms Agg Other bucket helper', () => { return new AggConfigs(indexPattern, [...aggs], { typesRegistry }); }; - beforeEach(() => { - mockDataServices(); - }); - describe('buildOtherBucketAgg', () => { test('returns a function', () => { const aggConfigs = getAggConfigs(singleTerm.aggs); diff --git a/src/plugins/data/public/search/aggs/buckets/_terms_other_bucket_helper.ts b/src/plugins/data/public/search/aggs/buckets/_terms_other_bucket_helper.ts index 4fd988e7b7e66..abda6b5fc5980 100644 --- a/src/plugins/data/public/search/aggs/buckets/_terms_other_bucket_helper.ts +++ b/src/plugins/data/public/search/aggs/buckets/_terms_other_bucket_helper.ts @@ -21,7 +21,7 @@ import { isNumber, keys, values, find, each, cloneDeep, flatten } from 'lodash'; import { buildExistsFilter, buildPhrasesFilter, buildQueryFromFilters } from '../../../../common'; import { AggGroupNames } from '../agg_groups'; import { IAggConfigs } from '../agg_configs'; -import { IBucketAggConfig } from './_bucket_agg_type'; +import { IBucketAggConfig } from './bucket_agg_type'; /** * walks the aggregation DSL and returns DSL starting at aggregation with id of startFromAggId diff --git a/src/plugins/data/public/search/aggs/buckets/_bucket_agg_type.ts b/src/plugins/data/public/search/aggs/buckets/bucket_agg_type.ts similarity index 86% rename from src/plugins/data/public/search/aggs/buckets/_bucket_agg_type.ts rename to src/plugins/data/public/search/aggs/buckets/bucket_agg_type.ts index 03629c3189cbb..f3c95b444dee9 100644 --- a/src/plugins/data/public/search/aggs/buckets/_bucket_agg_type.ts +++ b/src/plugins/data/public/search/aggs/buckets/bucket_agg_type.ts @@ -21,6 +21,7 @@ import { IAggConfig } from '../agg_config'; import { KBN_FIELD_TYPES } from '../../../../common'; import { AggType, AggTypeConfig } from '../agg_type'; import { AggParamType } from '../param_types/agg'; +import { GetInternalStartServicesFn } from '../../../types'; export interface IBucketAggConfig extends IAggConfig { type: InstanceType; @@ -39,6 +40,10 @@ interface BucketAggTypeConfig getKey?: (bucket: any, key: any, agg: IAggConfig) => any; } +interface BucketAggTypeDependencies { + getInternalStartServices: GetInternalStartServicesFn; +} + export class BucketAggType extends AggType< TBucketAggConfig, BucketAggParam @@ -46,8 +51,11 @@ export class BucketAggType any; type = bucketType; - constructor(config: BucketAggTypeConfig) { - super(config); + constructor( + config: BucketAggTypeConfig, + dependencies: BucketAggTypeDependencies + ) { + super(config, dependencies); this.getKey = config.getKey || diff --git a/src/plugins/data/public/search/aggs/buckets/create_filter/date_histogram.test.ts b/src/plugins/data/public/search/aggs/buckets/create_filter/date_histogram.test.ts index def354c4557cb..97c940b4ff4b1 100644 --- a/src/plugins/data/public/search/aggs/buckets/create_filter/date_histogram.test.ts +++ b/src/plugins/data/public/search/aggs/buckets/create_filter/date_histogram.test.ts @@ -29,8 +29,9 @@ import { } from '../date_histogram'; import { BUCKET_TYPES } from '../bucket_agg_types'; import { RangeFilter } from '../../../../../common'; -import { coreMock } from '../../../../../../../core/public/mocks'; +import { coreMock, notificationServiceMock } from '../../../../../../../core/public/mocks'; import { queryServiceMock } from '../../../../query/mocks'; +import { fieldFormatsServiceMock } from '../../../../field_formats/mocks'; describe('AggConfig Filters', () => { describe('date_histogram', () => { @@ -46,6 +47,10 @@ describe('AggConfig Filters', () => { aggTypesDependencies = { uiSettings, query: queryServiceMock.createSetupContract(), + getInternalStartServices: () => ({ + fieldFormats: fieldFormatsServiceMock.createStartContract(), + notifications: notificationServiceMock.createStartContract(), + }), }; mockDataServices(); @@ -90,7 +95,7 @@ describe('AggConfig Filters', () => { filter = createFilterDateHistogram(agg, bucketKey); }; - it('creates a valid range filter', () => { + test('creates a valid range filter', () => { init(); expect(filter).toHaveProperty('range'); @@ -110,7 +115,7 @@ describe('AggConfig Filters', () => { expect(filter.meta).toHaveProperty('index', '1234'); }); - it('extends the filter edge to 1ms before the next bucket for all interval options', () => { + test('extends the filter edge to 1ms before the next bucket for all interval options', () => { intervalOptions.forEach(option => { let duration; if (option.val !== 'custom' && moment(1, option.val).isValid()) { diff --git a/src/plugins/data/public/search/aggs/buckets/create_filter/date_range.test.ts b/src/plugins/data/public/search/aggs/buckets/create_filter/date_range.test.ts index 6a03176959a83..8c0466b769a7e 100644 --- a/src/plugins/data/public/search/aggs/buckets/create_filter/date_range.test.ts +++ b/src/plugins/data/public/search/aggs/buckets/create_filter/date_range.test.ts @@ -25,8 +25,9 @@ import { DateFormat } from '../../../../field_formats'; import { AggConfigs } from '../../agg_configs'; import { mockAggTypesRegistry } from '../../test_helpers'; import { BUCKET_TYPES } from '../bucket_agg_types'; -import { IBucketAggConfig } from '../_bucket_agg_type'; -import { coreMock } from '../../../../../../../core/public/mocks'; +import { IBucketAggConfig } from '../bucket_agg_type'; +import { coreMock, notificationServiceMock } from '../../../../../../../core/public/mocks'; +import { fieldFormatsServiceMock } from '../../../../field_formats/mocks'; describe('AggConfig Filters', () => { describe('Date range', () => { @@ -37,6 +38,10 @@ describe('AggConfig Filters', () => { aggTypesDependencies = { uiSettings, + getInternalStartServices: () => ({ + fieldFormats: fieldFormatsServiceMock.createStartContract(), + notifications: notificationServiceMock.createStartContract(), + }), }; }); @@ -71,7 +76,7 @@ describe('AggConfig Filters', () => { ); }; - it('should return a range filter for date_range agg', () => { + test('should return a range filter for date_range agg', () => { const aggConfigs = getAggConfigs(); const from = new Date('1 Feb 2015'); const to = new Date('7 Feb 2015'); diff --git a/src/plugins/data/public/search/aggs/buckets/create_filter/date_range.ts b/src/plugins/data/public/search/aggs/buckets/create_filter/date_range.ts index 9bfded0ce9729..118e9b26e87d5 100644 --- a/src/plugins/data/public/search/aggs/buckets/create_filter/date_range.ts +++ b/src/plugins/data/public/search/aggs/buckets/create_filter/date_range.ts @@ -18,7 +18,7 @@ */ import moment from 'moment'; -import { IBucketAggConfig } from '../_bucket_agg_type'; +import { IBucketAggConfig } from '../bucket_agg_type'; import { DateRangeKey } from '../lib/date_range'; import { buildRangeFilter, RangeFilterParams } from '../../../../../common'; diff --git a/src/plugins/data/public/search/aggs/buckets/create_filter/filters.test.ts b/src/plugins/data/public/search/aggs/buckets/create_filter/filters.test.ts index 32ada8d57c768..f5a0b5a7b9094 100644 --- a/src/plugins/data/public/search/aggs/buckets/create_filter/filters.test.ts +++ b/src/plugins/data/public/search/aggs/buckets/create_filter/filters.test.ts @@ -21,8 +21,9 @@ import { getFiltersBucketAgg, FiltersBucketAggDependencies } from '../filters'; import { createFilterFilters } from './filters'; import { AggConfigs } from '../../agg_configs'; import { mockAggTypesRegistry } from '../../test_helpers'; -import { IBucketAggConfig } from '../_bucket_agg_type'; -import { coreMock } from '../../../../../../../core/public/mocks'; +import { IBucketAggConfig } from '../bucket_agg_type'; +import { coreMock, notificationServiceMock } from '../../../../../../../core/public/mocks'; +import { fieldFormatsServiceMock } from '../../../../field_formats/mocks'; describe('AggConfig Filters', () => { describe('filters', () => { @@ -33,6 +34,10 @@ describe('AggConfig Filters', () => { aggTypesDependencies = { uiSettings, + getInternalStartServices: () => ({ + fieldFormats: fieldFormatsServiceMock.createStartContract(), + notifications: notificationServiceMock.createStartContract(), + }), }; }); @@ -67,7 +72,8 @@ describe('AggConfig Filters', () => { { typesRegistry: mockAggTypesRegistry([getFiltersBucketAgg(aggTypesDependencies)]) } ); }; - it('should return a filters filter', () => { + + test('should return a filters filter', () => { const aggConfigs = getAggConfigs(); const filter = createFilterFilters(aggConfigs.aggs[0] as IBucketAggConfig, 'type:nginx'); diff --git a/src/plugins/data/public/search/aggs/buckets/create_filter/filters.ts b/src/plugins/data/public/search/aggs/buckets/create_filter/filters.ts index 3b568d805f7c0..1999b759a23d0 100644 --- a/src/plugins/data/public/search/aggs/buckets/create_filter/filters.ts +++ b/src/plugins/data/public/search/aggs/buckets/create_filter/filters.ts @@ -18,7 +18,7 @@ */ import { get } from 'lodash'; -import { IBucketAggConfig } from '../_bucket_agg_type'; +import { IBucketAggConfig } from '../bucket_agg_type'; import { buildQueryFilter } from '../../../../../common'; export const createFilterFilters = (aggConfig: IBucketAggConfig, key: string) => { diff --git a/src/plugins/data/public/search/aggs/buckets/create_filter/histogram.test.ts b/src/plugins/data/public/search/aggs/buckets/create_filter/histogram.test.ts index dc8414d80c024..18b388be74877 100644 --- a/src/plugins/data/public/search/aggs/buckets/create_filter/histogram.test.ts +++ b/src/plugins/data/public/search/aggs/buckets/create_filter/histogram.test.ts @@ -19,19 +19,13 @@ import { createFilterHistogram } from './histogram'; import { AggConfigs } from '../../agg_configs'; -import { mockDataServices, mockAggTypesRegistry } from '../../test_helpers'; +import { mockAggTypesRegistry } from '../../test_helpers'; import { BUCKET_TYPES } from '../bucket_agg_types'; -import { IBucketAggConfig } from '../_bucket_agg_type'; +import { IBucketAggConfig } from '../bucket_agg_type'; import { BytesFormat, FieldFormatsGetConfigFn } from '../../../../../common'; describe('AggConfig Filters', () => { describe('histogram', () => { - beforeEach(() => { - mockDataServices(); - }); - - const typesRegistry = mockAggTypesRegistry(); - const getConfig = (() => {}) as FieldFormatsGetConfigFn; const getAggConfigs = () => { const field = { @@ -61,11 +55,11 @@ describe('AggConfig Filters', () => { }, }, ], - { typesRegistry } + { typesRegistry: mockAggTypesRegistry() } ); }; - it('should return an range filter for histogram', () => { + test('should return an range filter for histogram', () => { const aggConfigs = getAggConfigs(); const filter = createFilterHistogram(aggConfigs.aggs[0] as IBucketAggConfig, '2048'); diff --git a/src/plugins/data/public/search/aggs/buckets/create_filter/histogram.ts b/src/plugins/data/public/search/aggs/buckets/create_filter/histogram.ts index d4c00a0991fe2..f8e7747d49147 100644 --- a/src/plugins/data/public/search/aggs/buckets/create_filter/histogram.ts +++ b/src/plugins/data/public/search/aggs/buckets/create_filter/histogram.ts @@ -17,7 +17,7 @@ * under the License. */ -import { IBucketAggConfig } from '../_bucket_agg_type'; +import { IBucketAggConfig } from '../bucket_agg_type'; import { buildRangeFilter, RangeFilterParams } from '../../../../../common'; export const createFilterHistogram = (aggConfig: IBucketAggConfig, key: string) => { diff --git a/src/plugins/data/public/search/aggs/buckets/create_filter/ip_range.test.ts b/src/plugins/data/public/search/aggs/buckets/create_filter/ip_range.test.ts index ca51094da2f58..b528313b080d0 100644 --- a/src/plugins/data/public/search/aggs/buckets/create_filter/ip_range.test.ts +++ b/src/plugins/data/public/search/aggs/buckets/create_filter/ip_range.test.ts @@ -17,17 +17,26 @@ * under the License. */ -import { ipRangeBucketAgg } from '../ip_range'; +import { getIpRangeBucketAgg } from '../ip_range'; import { createFilterIpRange } from './ip_range'; import { AggConfigs, CreateAggConfigParams } from '../../agg_configs'; import { mockAggTypesRegistry } from '../../test_helpers'; import { IpFormat } from '../../../../../common'; import { BUCKET_TYPES } from '../bucket_agg_types'; -import { IBucketAggConfig } from '../_bucket_agg_type'; +import { IBucketAggConfig } from '../bucket_agg_type'; +import { fieldFormatsServiceMock } from '../../../../field_formats/mocks'; +import { notificationServiceMock } from '../../../../../../../core/public/mocks'; describe('AggConfig Filters', () => { describe('IP range', () => { - const typesRegistry = mockAggTypesRegistry([ipRangeBucketAgg]); + const typesRegistry = mockAggTypesRegistry([ + getIpRangeBucketAgg({ + getInternalStartServices: () => ({ + fieldFormats: fieldFormatsServiceMock.createStartContract(), + notifications: notificationServiceMock.createStartContract(), + }), + }), + ]); const getAggConfigs = (aggs: CreateAggConfigParams[]) => { const field = { name: 'ip', @@ -46,7 +55,7 @@ describe('AggConfig Filters', () => { return new AggConfigs(indexPattern, aggs, { typesRegistry }); }; - it('should return a range filter for ip_range agg', () => { + test('should return a range filter for ip_range agg', () => { const aggConfigs = getAggConfigs([ { type: BUCKET_TYPES.IP_RANGE, @@ -75,7 +84,7 @@ describe('AggConfig Filters', () => { expect(filter.range.ip).toHaveProperty('lte', '1.1.1.1'); }); - it('should return a range filter for ip_range agg using a CIDR mask', () => { + test('should return a range filter for ip_range agg using a CIDR mask', () => { const aggConfigs = getAggConfigs([ { type: BUCKET_TYPES.IP_RANGE, diff --git a/src/plugins/data/public/search/aggs/buckets/create_filter/ip_range.ts b/src/plugins/data/public/search/aggs/buckets/create_filter/ip_range.ts index 2d34c45aaab9d..aae212783b873 100644 --- a/src/plugins/data/public/search/aggs/buckets/create_filter/ip_range.ts +++ b/src/plugins/data/public/search/aggs/buckets/create_filter/ip_range.ts @@ -18,7 +18,7 @@ */ import { CidrMask } from '../lib/cidr_mask'; -import { IBucketAggConfig } from '../_bucket_agg_type'; +import { IBucketAggConfig } from '../bucket_agg_type'; import { IpRangeKey } from '../lib/ip_range'; import { buildRangeFilter, RangeFilterParams } from '../../../../../common'; diff --git a/src/plugins/data/public/search/aggs/buckets/create_filter/range.test.ts b/src/plugins/data/public/search/aggs/buckets/create_filter/range.test.ts index 3a6f8b36a9d96..14a7538aa95a4 100644 --- a/src/plugins/data/public/search/aggs/buckets/create_filter/range.test.ts +++ b/src/plugins/data/public/search/aggs/buckets/create_filter/range.test.ts @@ -17,22 +17,31 @@ * under the License. */ -import { rangeBucketAgg } from '../range'; +import { getRangeBucketAgg, RangeBucketAggDependencies } from '../range'; import { createFilterRange } from './range'; import { BytesFormat, FieldFormatsGetConfigFn } from '../../../../../common'; import { AggConfigs } from '../../agg_configs'; import { mockDataServices, mockAggTypesRegistry } from '../../test_helpers'; import { BUCKET_TYPES } from '../bucket_agg_types'; -import { IBucketAggConfig } from '../_bucket_agg_type'; +import { IBucketAggConfig } from '../bucket_agg_type'; +import { fieldFormatsServiceMock } from '../../../../field_formats/mocks'; +import { notificationServiceMock } from '../../../../../../../core/public/mocks'; describe('AggConfig Filters', () => { describe('range', () => { + let aggTypesDependencies: RangeBucketAggDependencies; + beforeEach(() => { + aggTypesDependencies = { + getInternalStartServices: () => ({ + fieldFormats: fieldFormatsServiceMock.createStartContract(), + notifications: notificationServiceMock.createStartContract(), + }), + }; + mockDataServices(); }); - const typesRegistry = mockAggTypesRegistry([rangeBucketAgg]); - const getConfig = (() => {}) as FieldFormatsGetConfigFn; const getAggConfigs = () => { const field = { @@ -62,11 +71,11 @@ describe('AggConfig Filters', () => { }, }, ], - { typesRegistry } + { typesRegistry: mockAggTypesRegistry([getRangeBucketAgg(aggTypesDependencies)]) } ); }; - it('should return a range filter for range agg', () => { + test('should return a range filter for range agg', () => { const aggConfigs = getAggConfigs(); const filter = createFilterRange(aggConfigs.aggs[0] as IBucketAggConfig, { gte: 1024, diff --git a/src/plugins/data/public/search/aggs/buckets/create_filter/range.ts b/src/plugins/data/public/search/aggs/buckets/create_filter/range.ts index d3d85f2441a8b..cbad8742bfab6 100644 --- a/src/plugins/data/public/search/aggs/buckets/create_filter/range.ts +++ b/src/plugins/data/public/search/aggs/buckets/create_filter/range.ts @@ -17,7 +17,7 @@ * under the License. */ -import { IBucketAggConfig } from '../_bucket_agg_type'; +import { IBucketAggConfig } from '../bucket_agg_type'; import { buildRangeFilter } from '../../../../../common'; export const createFilterRange = (aggConfig: IBucketAggConfig, params: any) => { diff --git a/src/plugins/data/public/search/aggs/buckets/create_filter/terms.test.ts b/src/plugins/data/public/search/aggs/buckets/create_filter/terms.test.ts index 511af450b0113..c11a7d1a4e6b8 100644 --- a/src/plugins/data/public/search/aggs/buckets/create_filter/terms.test.ts +++ b/src/plugins/data/public/search/aggs/buckets/create_filter/terms.test.ts @@ -17,17 +17,30 @@ * under the License. */ -import { termsBucketAgg } from '../terms'; +import { getTermsBucketAgg } from '../terms'; import { createFilterTerms } from './terms'; import { AggConfigs, CreateAggConfigParams } from '../../agg_configs'; import { mockAggTypesRegistry } from '../../test_helpers'; import { BUCKET_TYPES } from '../bucket_agg_types'; -import { IBucketAggConfig } from '../_bucket_agg_type'; +import { IBucketAggConfig } from '../bucket_agg_type'; import { Filter, ExistsFilter } from '../../../../../common'; +import { RangeBucketAggDependencies } from '../range'; +import { fieldFormatsServiceMock } from '../../../../field_formats/mocks'; +import { notificationServiceMock } from '../../../../../../../core/public/mocks'; describe('AggConfig Filters', () => { describe('terms', () => { - const typesRegistry = mockAggTypesRegistry([termsBucketAgg]); + let aggTypesDependencies: RangeBucketAggDependencies; + + beforeEach(() => { + aggTypesDependencies = { + getInternalStartServices: () => ({ + fieldFormats: fieldFormatsServiceMock.createStartContract(), + notifications: notificationServiceMock.createStartContract(), + }), + }; + }); + const getAggConfigs = (aggs: CreateAggConfigParams[]) => { const indexPattern = { id: '1234', @@ -43,10 +56,12 @@ describe('AggConfig Filters', () => { indexPattern, }; - return new AggConfigs(indexPattern, aggs, { typesRegistry }); + return new AggConfigs(indexPattern, aggs, { + typesRegistry: mockAggTypesRegistry([getTermsBucketAgg(aggTypesDependencies)]), + }); }; - it('should return a match_phrase filter for terms', () => { + test('should return a match_phrase filter for terms', () => { const aggConfigs = getAggConfigs([ { type: BUCKET_TYPES.TERMS, schema: 'segment', params: { field: 'field' } }, ]); @@ -65,7 +80,7 @@ describe('AggConfig Filters', () => { expect(filter.meta).toHaveProperty('index', '1234'); }); - it('should set query to true or false for boolean filter', () => { + test('should set query to true or false for boolean filter', () => { const aggConfigs = getAggConfigs([ { type: BUCKET_TYPES.TERMS, schema: 'segment', params: { field: 'field' } }, ]); @@ -93,7 +108,7 @@ describe('AggConfig Filters', () => { expect(filterTrue.query.match_phrase.field).toBeTruthy(); }); - it('should generate correct __missing__ filter', () => { + test('should generate correct __missing__ filter', () => { const aggConfigs = getAggConfigs([ { type: BUCKET_TYPES.TERMS, schema: 'segment', params: { field: 'field' } }, ]); @@ -110,7 +125,7 @@ describe('AggConfig Filters', () => { expect(filter.meta).toHaveProperty('negate', true); }); - it('should generate correct __other__ filter', () => { + test('should generate correct __other__ filter', () => { const aggConfigs = getAggConfigs([ { type: BUCKET_TYPES.TERMS, schema: 'segment', params: { field: 'field' } }, ]); diff --git a/src/plugins/data/public/search/aggs/buckets/create_filter/terms.ts b/src/plugins/data/public/search/aggs/buckets/create_filter/terms.ts index 43ebfc0e90db2..95de19b96abd4 100644 --- a/src/plugins/data/public/search/aggs/buckets/create_filter/terms.ts +++ b/src/plugins/data/public/search/aggs/buckets/create_filter/terms.ts @@ -17,7 +17,7 @@ * under the License. */ -import { IBucketAggConfig } from '../_bucket_agg_type'; +import { IBucketAggConfig } from '../bucket_agg_type'; import { buildPhrasesFilter, buildExistsFilter, diff --git a/src/plugins/data/public/search/aggs/buckets/date_histogram.ts b/src/plugins/data/public/search/aggs/buckets/date_histogram.ts index 7701f1bbcb4d0..e6fd259fabc92 100644 --- a/src/plugins/data/public/search/aggs/buckets/date_histogram.ts +++ b/src/plugins/data/public/search/aggs/buckets/date_histogram.ts @@ -23,7 +23,7 @@ import { i18n } from '@kbn/i18n'; import { IUiSettingsClient } from 'src/core/public'; import { TimeBuckets } from './lib/time_buckets'; -import { BucketAggType, IBucketAggConfig } from './_bucket_agg_type'; +import { BucketAggType, IBucketAggConfig } from './bucket_agg_type'; import { BUCKET_TYPES } from './bucket_agg_types'; import { createFilterDateHistogram } from './create_filter/date_histogram'; import { intervalOptions } from './_interval_options'; @@ -33,8 +33,8 @@ import { isMetricAggType } from '../metrics/metric_agg_type'; import { FIELD_FORMAT_IDS, KBN_FIELD_TYPES } from '../../../../common'; import { TimefilterContract } from '../../../query'; -import { getFieldFormats } from '../../../../public/services'; import { QuerySetup } from '../../../query/query_service'; +import { GetInternalStartServicesFn } from '../../../types'; const detectedTimezone = moment.tz.guess(); const tzOffset = moment().format('Z'); @@ -61,6 +61,7 @@ interface ITimeBuckets { export interface DateHistogramBucketAggDependencies { uiSettings: IUiSettingsClient; query: QuerySetup; + getInternalStartServices: GetInternalStartServicesFn; } export interface IBucketDateHistogramAggConfig extends IBucketAggConfig { @@ -74,211 +75,218 @@ export function isDateHistogramBucketAggConfig(agg: any): agg is IBucketDateHist export const getDateHistogramBucketAgg = ({ uiSettings, query, + getInternalStartServices, }: DateHistogramBucketAggDependencies) => - new BucketAggType({ - name: BUCKET_TYPES.DATE_HISTOGRAM, - title: i18n.translate('data.search.aggs.buckets.dateHistogramTitle', { - defaultMessage: 'Date Histogram', - }), - ordered: { - date: true, - }, - makeLabel(agg) { - let output: Record = {}; + new BucketAggType( + { + name: BUCKET_TYPES.DATE_HISTOGRAM, + title: i18n.translate('data.search.aggs.buckets.dateHistogramTitle', { + defaultMessage: 'Date Histogram', + }), + ordered: { + date: true, + }, + makeLabel(agg) { + let output: Record = {}; - if (this.params) { - output = writeParams(this.params, agg); - } + if (this.params) { + output = writeParams(this.params, agg); + } - const field = agg.getFieldDisplayName(); - return i18n.translate('data.search.aggs.buckets.dateHistogramLabel', { - defaultMessage: '{fieldName} per {intervalDescription}', - values: { - fieldName: field, - intervalDescription: output.metricScaleText || output.bucketInterval.description, - }, - }); - }, - createFilter: createFilterDateHistogram, - decorateAggConfig() { - let buckets: any; + const field = agg.getFieldDisplayName(); + return i18n.translate('data.search.aggs.buckets.dateHistogramLabel', { + defaultMessage: '{fieldName} per {intervalDescription}', + values: { + fieldName: field, + intervalDescription: output.metricScaleText || output.bucketInterval.description, + }, + }); + }, + createFilter: createFilterDateHistogram, + decorateAggConfig() { + let buckets: any; - return { - buckets: { - configurable: true, - get() { - if (buckets) return buckets; + return { + buckets: { + configurable: true, + get() { + if (buckets) return buckets; - const { timefilter } = query.timefilter; - buckets = new TimeBuckets({ uiSettings }); - updateTimeBuckets(this, timefilter, buckets); + const { timefilter } = query.timefilter; + buckets = new TimeBuckets({ uiSettings }); + updateTimeBuckets(this, timefilter, buckets); - return buckets; - }, - } as any, - }; - }, - getFormat(agg) { - const DateFieldFormat = getFieldFormats().getType(FIELD_FORMAT_IDS.DATE); + return buckets; + }, + } as any, + }; + }, + getFormat(agg) { + const { fieldFormats } = getInternalStartServices(); + const DateFieldFormat = fieldFormats.getType(FIELD_FORMAT_IDS.DATE); - if (!DateFieldFormat) { - throw new Error('Unable to retrieve Date Field Format'); - } + if (!DateFieldFormat) { + throw new Error('Unable to retrieve Date Field Format'); + } - return new DateFieldFormat( + return new DateFieldFormat( + { + pattern: agg.buckets.getScaledDateFormat(), + }, + (key: string) => uiSettings.get(key) + ); + }, + params: [ { - pattern: agg.buckets.getScaledDateFormat(), + name: 'field', + type: 'field', + filterFieldTypes: KBN_FIELD_TYPES.DATE, + default(agg: IBucketDateHistogramAggConfig) { + return agg.getIndexPattern().timeFieldName; + }, + onChange(agg: IBucketDateHistogramAggConfig) { + if (get(agg, 'params.interval') === 'auto' && !agg.fieldIsTimeField()) { + delete agg.params.interval; + } + }, }, - (key: string) => uiSettings.get(key) - ); - }, - params: [ - { - name: 'field', - type: 'field', - filterFieldTypes: KBN_FIELD_TYPES.DATE, - default(agg: IBucketDateHistogramAggConfig) { - return agg.getIndexPattern().timeFieldName; + { + name: 'timeRange', + default: null, + write: noop, }, - onChange(agg: IBucketDateHistogramAggConfig) { - if (get(agg, 'params.interval') === 'auto' && !agg.fieldIsTimeField()) { - delete agg.params.interval; - } + { + name: 'useNormalizedEsInterval', + default: true, + write: noop, }, - }, - { - name: 'timeRange', - default: null, - write: noop, - }, - { - name: 'useNormalizedEsInterval', - default: true, - write: noop, - }, - { - name: 'scaleMetricValues', - default: false, - write: noop, - advanced: true, - }, - { - name: 'interval', - deserialize(state: any, agg) { - // For upgrading from 7.0.x to 7.1.x - intervals are now stored as key of options or custom value - if (state === 'custom') { - return get(agg, 'params.customInterval'); - } + { + name: 'scaleMetricValues', + default: false, + write: noop, + advanced: true, + }, + { + name: 'interval', + deserialize(state: any, agg) { + // For upgrading from 7.0.x to 7.1.x - intervals are now stored as key of options or custom value + if (state === 'custom') { + return get(agg, 'params.customInterval'); + } - const interval = find(intervalOptions, { val: state }); + const interval = find(intervalOptions, { val: state }); - // For upgrading from 4.0.x to 4.1.x - intervals are now stored as 'y' instead of 'year', - // but this maps the old values to the new values - if (!interval && state === 'year') { - return 'y'; - } - return state; - }, - default: 'auto', - options: intervalOptions, - write(agg, output, aggs) { - const { timefilter } = query.timefilter; - updateTimeBuckets(agg, timefilter); + // For upgrading from 4.0.x to 4.1.x - intervals are now stored as 'y' instead of 'year', + // but this maps the old values to the new values + if (!interval && state === 'year') { + return 'y'; + } + return state; + }, + default: 'auto', + options: intervalOptions, + write(agg, output, aggs) { + const { timefilter } = query.timefilter; + updateTimeBuckets(agg, timefilter); - const { useNormalizedEsInterval, scaleMetricValues } = agg.params; - const interval = agg.buckets.getInterval(useNormalizedEsInterval); - output.bucketInterval = interval; - if (interval.expression === '0ms') { - // We are hitting this code a couple of times while configuring in editor - // with an interval of 0ms because the overall time range has not yet been - // set. Since 0ms is not a valid ES interval, we cannot pass it through dateHistogramInterval - // below, since it would throw an exception. So in the cases we still have an interval of 0ms - // here we simply skip the rest of the method and never write an interval into the DSL, since - // this DSL will anyway not be used before we're passing this code with an actual interval. - return; - } - output.params = { - ...output.params, - ...dateHistogramInterval(interval.expression), - }; + const { useNormalizedEsInterval, scaleMetricValues } = agg.params; + const interval = agg.buckets.getInterval(useNormalizedEsInterval); + output.bucketInterval = interval; + if (interval.expression === '0ms') { + // We are hitting this code a couple of times while configuring in editor + // with an interval of 0ms because the overall time range has not yet been + // set. Since 0ms is not a valid ES interval, we cannot pass it through dateHistogramInterval + // below, since it would throw an exception. So in the cases we still have an interval of 0ms + // here we simply skip the rest of the method and never write an interval into the DSL, since + // this DSL will anyway not be used before we're passing this code with an actual interval. + return; + } + output.params = { + ...output.params, + ...dateHistogramInterval(interval.expression), + }; - const scaleMetrics = scaleMetricValues && interval.scaled && interval.scale < 1; - if (scaleMetrics && aggs) { - const metrics = aggs.aggs.filter(a => isMetricAggType(a.type)); - const all = every(metrics, (a: IBucketAggConfig) => { - const { type } = a; + const scaleMetrics = scaleMetricValues && interval.scaled && interval.scale < 1; + if (scaleMetrics && aggs) { + const metrics = aggs.aggs.filter(a => isMetricAggType(a.type)); + const all = every(metrics, (a: IBucketAggConfig) => { + const { type } = a; - if (isMetricAggType(type)) { - return type.isScalable(); + if (isMetricAggType(type)) { + return type.isScalable(); + } + }); + if (all) { + output.metricScale = interval.scale; + output.metricScaleText = interval.preScaled.description; } - }); - if (all) { - output.metricScale = interval.scale; - output.metricScaleText = interval.preScaled.description; } - } + }, }, - }, - { - name: 'time_zone', - default: undefined, - // We don't ever want this parameter to be serialized out (when saving or to URLs) - // since we do all the logic handling it "on the fly" in the `write` method, to prevent - // time_zones being persisted into saved_objects - serialize: noop, - write(agg, output) { - // If a time_zone has been set explicitly always prefer this. - let tz = agg.params.time_zone; - if (!tz && agg.params.field) { - // If a field has been configured check the index pattern's typeMeta if a date_histogram on that - // field requires a specific time_zone - tz = get(agg.getIndexPattern(), [ - 'typeMeta', - 'aggs', - 'date_histogram', - agg.params.field.name, - 'time_zone', - ]); - } - if (!tz) { - // If the index pattern typeMeta data, didn't had a time zone assigned for the selected field use the configured tz - const isDefaultTimezone = uiSettings.isDefault('dateFormat:tz'); - tz = isDefaultTimezone ? detectedTimezone || tzOffset : uiSettings.get('dateFormat:tz'); - } - output.params.time_zone = tz; + { + name: 'time_zone', + default: undefined, + // We don't ever want this parameter to be serialized out (when saving or to URLs) + // since we do all the logic handling it "on the fly" in the `write` method, to prevent + // time_zones being persisted into saved_objects + serialize: noop, + write(agg, output) { + // If a time_zone has been set explicitly always prefer this. + let tz = agg.params.time_zone; + if (!tz && agg.params.field) { + // If a field has been configured check the index pattern's typeMeta if a date_histogram on that + // field requires a specific time_zone + tz = get(agg.getIndexPattern(), [ + 'typeMeta', + 'aggs', + 'date_histogram', + agg.params.field.name, + 'time_zone', + ]); + } + if (!tz) { + // If the index pattern typeMeta data, didn't had a time zone assigned for the selected field use the configured tz + const isDefaultTimezone = uiSettings.isDefault('dateFormat:tz'); + tz = isDefaultTimezone + ? detectedTimezone || tzOffset + : uiSettings.get('dateFormat:tz'); + } + output.params.time_zone = tz; + }, }, - }, - { - name: 'drop_partials', - default: false, - write: noop, - shouldShow: agg => { - const field = agg.params.field; - return field && field.name && field.name === agg.getIndexPattern().timeFieldName; + { + name: 'drop_partials', + default: false, + write: noop, + shouldShow: agg => { + const field = agg.params.field; + return field && field.name && field.name === agg.getIndexPattern().timeFieldName; + }, }, - }, - { - name: 'format', - }, - { - name: 'min_doc_count', - default: 1, - }, - { - name: 'extended_bounds', - default: {}, - write(agg, output) { - const val = agg.params.extended_bounds; + { + name: 'format', + }, + { + name: 'min_doc_count', + default: 1, + }, + { + name: 'extended_bounds', + default: {}, + write(agg, output) { + const val = agg.params.extended_bounds; - if (val.min != null || val.max != null) { - output.params.extended_bounds = { - min: moment(val.min).valueOf(), - max: moment(val.max).valueOf(), - }; + if (val.min != null || val.max != null) { + output.params.extended_bounds = { + min: moment(val.min).valueOf(), + max: moment(val.max).valueOf(), + }; - return; - } + return; + } + }, }, - }, - ], - }); + ], + }, + { getInternalStartServices } + ); diff --git a/src/plugins/data/public/search/aggs/buckets/date_range.test.ts b/src/plugins/data/public/search/aggs/buckets/date_range.test.ts index 4ea550492fa09..c050620c3a856 100644 --- a/src/plugins/data/public/search/aggs/buckets/date_range.test.ts +++ b/src/plugins/data/public/search/aggs/buckets/date_range.test.ts @@ -17,11 +17,12 @@ * under the License. */ -import { coreMock } from '../../../../../../../src/core/public/mocks'; +import { coreMock, notificationServiceMock } from '../../../../../../../src/core/public/mocks'; import { getDateRangeBucketAgg, DateRangeBucketAggDependencies } from './date_range'; import { AggConfigs } from '../agg_configs'; import { mockAggTypesRegistry } from '../test_helpers'; import { BUCKET_TYPES } from './bucket_agg_types'; +import { fieldFormatsServiceMock } from '../../../field_formats/mocks'; describe('date_range params', () => { let aggTypesDependencies: DateRangeBucketAggDependencies; @@ -31,6 +32,10 @@ describe('date_range params', () => { aggTypesDependencies = { uiSettings, + getInternalStartServices: () => ({ + fieldFormats: fieldFormatsServiceMock.createStartContract(), + notifications: notificationServiceMock.createStartContract(), + }), }; }); diff --git a/src/plugins/data/public/search/aggs/buckets/date_range.ts b/src/plugins/data/public/search/aggs/buckets/date_range.ts index 8133a47ec7248..07d927e64a943 100644 --- a/src/plugins/data/public/search/aggs/buckets/date_range.ts +++ b/src/plugins/data/public/search/aggs/buckets/date_range.ts @@ -23,12 +23,12 @@ import { i18n } from '@kbn/i18n'; import { IUiSettingsClient } from 'src/core/public'; import { BUCKET_TYPES } from './bucket_agg_types'; -import { BucketAggType, IBucketAggConfig } from './_bucket_agg_type'; +import { BucketAggType, IBucketAggConfig } from './bucket_agg_type'; import { createFilterDateRange } from './create_filter/date_range'; import { convertDateRangeToString, DateRangeKey } from './lib/date_range'; import { KBN_FIELD_TYPES, FieldFormat, TEXT_CONTEXT_TYPE } from '../../../../common'; -import { getFieldFormats } from '../../../../public/services'; +import { GetInternalStartServicesFn } from '../../../types'; const dateRangeTitle = i18n.translate('data.search.aggs.buckets.dateRangeTitle', { defaultMessage: 'Date Range', @@ -36,76 +36,85 @@ const dateRangeTitle = i18n.translate('data.search.aggs.buckets.dateRangeTitle', export interface DateRangeBucketAggDependencies { uiSettings: IUiSettingsClient; + getInternalStartServices: GetInternalStartServicesFn; } -export const getDateRangeBucketAgg = ({ uiSettings }: DateRangeBucketAggDependencies) => - new BucketAggType({ - name: BUCKET_TYPES.DATE_RANGE, - title: dateRangeTitle, - createFilter: createFilterDateRange, - getKey({ from, to }): DateRangeKey { - return { from, to }; - }, - getFormat(agg) { - const fieldFormatsService = getFieldFormats(); +export const getDateRangeBucketAgg = ({ + uiSettings, + getInternalStartServices, +}: DateRangeBucketAggDependencies) => + new BucketAggType( + { + name: BUCKET_TYPES.DATE_RANGE, + title: dateRangeTitle, + createFilter: createFilterDateRange, + getKey({ from, to }): DateRangeKey { + return { from, to }; + }, + getFormat(agg) { + const { fieldFormats } = getInternalStartServices(); - const formatter = agg.fieldOwnFormatter( - TEXT_CONTEXT_TYPE, - fieldFormatsService.getDefaultInstance(KBN_FIELD_TYPES.DATE) - ); - const DateRangeFormat = FieldFormat.from(function(range: DateRangeKey) { - return convertDateRangeToString(range, formatter); - }); - return new DateRangeFormat(); - }, - makeLabel(aggConfig) { - return aggConfig.getFieldDisplayName() + ' date ranges'; - }, - params: [ - { - name: 'field', - type: 'field', - filterFieldTypes: KBN_FIELD_TYPES.DATE, - default(agg: IBucketAggConfig) { - return agg.getIndexPattern().timeFieldName; - }, + const formatter = agg.fieldOwnFormatter( + TEXT_CONTEXT_TYPE, + fieldFormats.getDefaultInstance(KBN_FIELD_TYPES.DATE) + ); + const DateRangeFormat = FieldFormat.from(function(range: DateRangeKey) { + return convertDateRangeToString(range, formatter); + }); + return new DateRangeFormat(); }, - { - name: 'ranges', - default: [ - { - from: 'now-1w/w', - to: 'now', - }, - ], + makeLabel(aggConfig) { + return aggConfig.getFieldDisplayName() + ' date ranges'; }, - { - name: 'time_zone', - default: undefined, - // Implimentation method is the same as that of date_histogram - serialize: () => undefined, - write: (agg, output) => { - const field = agg.getParam('field'); - let tz = agg.getParam('time_zone'); + params: [ + { + name: 'field', + type: 'field', + filterFieldTypes: KBN_FIELD_TYPES.DATE, + default(agg: IBucketAggConfig) { + return agg.getIndexPattern().timeFieldName; + }, + }, + { + name: 'ranges', + default: [ + { + from: 'now-1w/w', + to: 'now', + }, + ], + }, + { + name: 'time_zone', + default: undefined, + // Implimentation method is the same as that of date_histogram + serialize: () => undefined, + write: (agg, output) => { + const field = agg.getParam('field'); + let tz = agg.getParam('time_zone'); - if (!tz && field) { - tz = get(agg.getIndexPattern(), [ - 'typeMeta', - 'aggs', - 'date_range', - field.name, - 'time_zone', - ]); - } - if (!tz) { - const detectedTimezone = moment.tz.guess(); - const tzOffset = moment().format('Z'); - const isDefaultTimezone = uiSettings.isDefault('dateFormat:tz'); + if (!tz && field) { + tz = get(agg.getIndexPattern(), [ + 'typeMeta', + 'aggs', + 'date_range', + field.name, + 'time_zone', + ]); + } + if (!tz) { + const detectedTimezone = moment.tz.guess(); + const tzOffset = moment().format('Z'); + const isDefaultTimezone = uiSettings.isDefault('dateFormat:tz'); - tz = isDefaultTimezone ? detectedTimezone || tzOffset : uiSettings.get('dateFormat:tz'); - } - output.params.time_zone = tz; + tz = isDefaultTimezone + ? detectedTimezone || tzOffset + : uiSettings.get('dateFormat:tz'); + } + output.params.time_zone = tz; + }, }, - }, - ], - }); + ], + }, + { getInternalStartServices } + ); diff --git a/src/plugins/data/public/search/aggs/buckets/filter.ts b/src/plugins/data/public/search/aggs/buckets/filter.ts index 80efc0cf92071..accbdf4dd783d 100644 --- a/src/plugins/data/public/search/aggs/buckets/filter.ts +++ b/src/plugins/data/public/search/aggs/buckets/filter.ts @@ -18,19 +18,28 @@ */ import { i18n } from '@kbn/i18n'; -import { BucketAggType } from './_bucket_agg_type'; +import { BucketAggType } from './bucket_agg_type'; import { BUCKET_TYPES } from './bucket_agg_types'; +import { GetInternalStartServicesFn } from '../../../types'; const filterTitle = i18n.translate('data.search.aggs.buckets.filterTitle', { defaultMessage: 'Filter', }); -export const filterBucketAgg = new BucketAggType({ - name: BUCKET_TYPES.FILTER, - title: filterTitle, - params: [ +export interface FilterBucketAggDependencies { + getInternalStartServices: GetInternalStartServicesFn; +} + +export const getFilterBucketAgg = ({ getInternalStartServices }: FilterBucketAggDependencies) => + new BucketAggType( { - name: 'geo_bounding_box', + name: BUCKET_TYPES.FILTER, + title: filterTitle, + params: [ + { + name: 'geo_bounding_box', + }, + ], }, - ], -}); + { getInternalStartServices } + ); diff --git a/src/plugins/data/public/search/aggs/buckets/filters.ts b/src/plugins/data/public/search/aggs/buckets/filters.ts index 8b9aca87f8735..a42cb70a62b7d 100644 --- a/src/plugins/data/public/search/aggs/buckets/filters.ts +++ b/src/plugins/data/public/search/aggs/buckets/filters.ts @@ -23,11 +23,12 @@ import { IUiSettingsClient } from 'src/core/public'; import { createFilterFilters } from './create_filter/filters'; import { toAngularJSON } from '../utils'; -import { BucketAggType } from './_bucket_agg_type'; +import { BucketAggType } from './bucket_agg_type'; import { BUCKET_TYPES } from './bucket_agg_types'; import { Storage } from '../../../../../../plugins/kibana_utils/public'; import { getEsQueryConfig, buildEsQuery, Query } from '../../../../common'; import { getQueryLog } from '../../../query'; +import { GetInternalStartServicesFn } from '../../../types'; const filtersTitle = i18n.translate('data.search.aggs.buckets.filtersTitle', { defaultMessage: 'Filters', @@ -43,69 +44,81 @@ interface FilterValue { export interface FiltersBucketAggDependencies { uiSettings: IUiSettingsClient; + getInternalStartServices: GetInternalStartServicesFn; } -export const getFiltersBucketAgg = ({ uiSettings }: FiltersBucketAggDependencies) => - new BucketAggType({ - name: BUCKET_TYPES.FILTERS, - title: filtersTitle, - createFilter: createFilterFilters, - customLabels: false, - params: [ - { - name: 'filters', - default: [ - { input: { query: '', language: uiSettings.get('search:queryLanguage') }, label: '' }, - ], - write(aggConfig, output) { - const inFilters: FilterValue[] = aggConfig.params.filters; - if (!size(inFilters)) return; +export const getFiltersBucketAgg = ({ + uiSettings, + getInternalStartServices, +}: FiltersBucketAggDependencies) => + new BucketAggType( + { + name: BUCKET_TYPES.FILTERS, + title: filtersTitle, + createFilter: createFilterFilters, + customLabels: false, + params: [ + { + name: 'filters', + default: [ + { input: { query: '', language: uiSettings.get('search:queryLanguage') }, label: '' }, + ], + write(aggConfig, output) { + const inFilters: FilterValue[] = aggConfig.params.filters; + if (!size(inFilters)) return; - inFilters.forEach(filter => { - const persistedLog = getQueryLog( - uiSettings, - new Storage(window.localStorage), - 'vis_default_editor', - filter.input.language - ); - persistedLog.add(filter.input.query); - }); + inFilters.forEach(filter => { + const persistedLog = getQueryLog( + uiSettings, + new Storage(window.localStorage), + 'vis_default_editor', + filter.input.language + ); + persistedLog.add(filter.input.query); + }); - const outFilters = transform( - inFilters, - function(filters, filter) { - const input = cloneDeep(filter.input); + const outFilters = transform( + inFilters, + function(filters, filter) { + const input = cloneDeep(filter.input); - if (!input) { - console.log('malformed filter agg params, missing "input" query'); // eslint-disable-line no-console - return; - } + if (!input) { + console.log('malformed filter agg params, missing "input" query'); // eslint-disable-line no-console + return; + } - const esQueryConfigs = getEsQueryConfig(uiSettings); - const query = buildEsQuery(aggConfig.getIndexPattern(), [input], [], esQueryConfigs); + const esQueryConfigs = getEsQueryConfig(uiSettings); + const query = buildEsQuery( + aggConfig.getIndexPattern(), + [input], + [], + esQueryConfigs + ); - if (!query) { - console.log('malformed filter agg params, missing "query" on input'); // eslint-disable-line no-console - return; - } + if (!query) { + console.log('malformed filter agg params, missing "query" on input'); // eslint-disable-line no-console + return; + } - const matchAllLabel = filter.input.query === '' ? '*' : ''; - const label = - filter.label || - matchAllLabel || - (typeof filter.input.query === 'string' - ? filter.input.query - : toAngularJSON(filter.input.query)); - filters[label] = { query }; - }, - {} - ); + const matchAllLabel = filter.input.query === '' ? '*' : ''; + const label = + filter.label || + matchAllLabel || + (typeof filter.input.query === 'string' + ? filter.input.query + : toAngularJSON(filter.input.query)); + filters[label] = { query }; + }, + {} + ); - if (!size(outFilters)) return; + if (!size(outFilters)) return; - const params = output.params || (output.params = {}); - params.filters = outFilters; + const params = output.params || (output.params = {}); + params.filters = outFilters; + }, }, - }, - ], - }); + ], + }, + { getInternalStartServices } + ); diff --git a/src/plugins/data/public/search/aggs/buckets/geo_hash.test.ts b/src/plugins/data/public/search/aggs/buckets/geo_hash.test.ts index 408cdf22bcbc2..24270dd33a576 100644 --- a/src/plugins/data/public/search/aggs/buckets/geo_hash.test.ts +++ b/src/plugins/data/public/search/aggs/buckets/geo_hash.test.ts @@ -17,13 +17,29 @@ * under the License. */ -import { geoHashBucketAgg } from './geo_hash'; +import { getGeoHashBucketAgg, GeoHashBucketAggDependencies } from './geo_hash'; import { AggConfigs, IAggConfigs } from '../agg_configs'; import { mockAggTypesRegistry } from '../test_helpers'; import { BUCKET_TYPES } from './bucket_agg_types'; -import { IBucketAggConfig } from './_bucket_agg_type'; +import { notificationServiceMock } from '../../../../../../../src/core/public/mocks'; +import { fieldFormatsServiceMock } from '../../../field_formats/mocks'; +import { BucketAggType, IBucketAggConfig } from './bucket_agg_type'; describe('Geohash Agg', () => { + let aggTypesDependencies: GeoHashBucketAggDependencies; + let geoHashBucketAgg: BucketAggType; + + beforeEach(() => { + aggTypesDependencies = { + getInternalStartServices: () => ({ + fieldFormats: fieldFormatsServiceMock.createStartContract(), + notifications: notificationServiceMock.createStartContract(), + }), + }; + + geoHashBucketAgg = getGeoHashBucketAgg(aggTypesDependencies); + }); + const getAggConfigs = (params?: Record) => { const indexPattern = { id: '1234', @@ -74,7 +90,7 @@ describe('Geohash Agg', () => { precisionParam = geoHashBucketAgg.params[PRECISION_PARAM_INDEX]; }); - it('should select precision parameter', () => { + test('should select precision parameter', () => { expect(precisionParam.name).toEqual('precision'); }); }); @@ -89,7 +105,7 @@ describe('Geohash Agg', () => { geoHashGridAgg = aggConfigs.aggs[0] as IBucketAggConfig; }); - it('should create filter, geohash_grid, and geo_centroid aggregations', () => { + test('should create filter, geohash_grid, and geo_centroid aggregations', () => { const requestAggs = geoHashBucketAgg.getRequestAggs(geoHashGridAgg) as IBucketAggConfig[]; expect(requestAggs.length).toEqual(3); @@ -101,7 +117,7 @@ describe('Geohash Agg', () => { }); describe('aggregation options', () => { - it('should only create geohash_grid and geo_centroid aggregations when isFilteredByCollar is false', () => { + test('should only create geohash_grid and geo_centroid aggregations when isFilteredByCollar is false', () => { const aggConfigs = getAggConfigs({ isFilteredByCollar: false }); const requestAggs = geoHashBucketAgg.getRequestAggs( aggConfigs.aggs[0] as IBucketAggConfig @@ -112,7 +128,7 @@ describe('Geohash Agg', () => { expect(requestAggs[1].type.name).toEqual('geo_centroid'); }); - it('should only create filter and geohash_grid aggregations when useGeocentroid is false', () => { + test('should only create filter and geohash_grid aggregations when useGeocentroid is false', () => { const aggConfigs = getAggConfigs({ useGeocentroid: false }); const requestAggs = geoHashBucketAgg.getRequestAggs( aggConfigs.aggs[0] as IBucketAggConfig @@ -138,7 +154,7 @@ describe('Geohash Agg', () => { ) as IBucketAggConfig[]; }); - it('should change geo_bounding_box filter aggregation and vis session state when map movement is outside map collar', () => { + test('should change geo_bounding_box filter aggregation and vis session state when map movement is outside map collar', () => { const [, geoBoxingBox] = geoHashBucketAgg.getRequestAggs( getAggConfigs({ boundingBox: { @@ -151,7 +167,7 @@ describe('Geohash Agg', () => { expect(originalRequestAggs[1].params).not.toEqual(geoBoxingBox.params); }); - it('should not change geo_bounding_box filter aggregation and vis session state when map movement is within map collar', () => { + test('should not change geo_bounding_box filter aggregation and vis session state when map movement is within map collar', () => { const [, geoBoxingBox] = geoHashBucketAgg.getRequestAggs( getAggConfigs({ boundingBox: { diff --git a/src/plugins/data/public/search/aggs/buckets/geo_hash.ts b/src/plugins/data/public/search/aggs/buckets/geo_hash.ts index 3ffec09a84387..eab10edad60f6 100644 --- a/src/plugins/data/public/search/aggs/buckets/geo_hash.ts +++ b/src/plugins/data/public/search/aggs/buckets/geo_hash.ts @@ -18,9 +18,10 @@ */ import { i18n } from '@kbn/i18n'; -import { BucketAggType, IBucketAggConfig } from './_bucket_agg_type'; +import { BucketAggType, IBucketAggConfig } from './bucket_agg_type'; import { KBN_FIELD_TYPES } from '../../../../common'; import { BUCKET_TYPES } from './bucket_agg_types'; +import { GetInternalStartServicesFn } from '../../../types'; const defaultBoundingBox = { top_left: { lat: 1, lon: 1 }, @@ -33,83 +34,91 @@ const geohashGridTitle = i18n.translate('data.search.aggs.buckets.geohashGridTit defaultMessage: 'Geohash', }); -export const geoHashBucketAgg = new BucketAggType({ - name: BUCKET_TYPES.GEOHASH_GRID, - title: geohashGridTitle, - params: [ - { - name: 'field', - type: 'field', - filterFieldTypes: KBN_FIELD_TYPES.GEO_POINT, - }, - { - name: 'autoPrecision', - default: true, - write: () => {}, - }, - { - name: 'precision', - default: defaultPrecision, - write(aggConfig, output) { - output.params.precision = aggConfig.params.precision; - }, - }, - { - name: 'useGeocentroid', - default: true, - write: () => {}, - }, - { - name: 'isFilteredByCollar', - default: true, - write: () => {}, - }, +export interface GeoHashBucketAggDependencies { + getInternalStartServices: GetInternalStartServicesFn; +} + +export const getGeoHashBucketAgg = ({ getInternalStartServices }: GeoHashBucketAggDependencies) => + new BucketAggType( { - name: 'boundingBox', - default: null, - write: () => {}, - }, - ], - getRequestAggs(agg) { - const aggs = []; - const params = agg.params; + name: BUCKET_TYPES.GEOHASH_GRID, + title: geohashGridTitle, + params: [ + { + name: 'field', + type: 'field', + filterFieldTypes: KBN_FIELD_TYPES.GEO_POINT, + }, + { + name: 'autoPrecision', + default: true, + write: () => {}, + }, + { + name: 'precision', + default: defaultPrecision, + write(aggConfig, output) { + output.params.precision = aggConfig.params.precision; + }, + }, + { + name: 'useGeocentroid', + default: true, + write: () => {}, + }, + { + name: 'isFilteredByCollar', + default: true, + write: () => {}, + }, + { + name: 'boundingBox', + default: null, + write: () => {}, + }, + ], + getRequestAggs(agg) { + const aggs = []; + const params = agg.params; - if (params.isFilteredByCollar && agg.getField()) { - aggs.push( - agg.aggConfigs.createAggConfig( - { - type: 'filter', - id: 'filter_agg', - enabled: true, - params: { - geo_bounding_box: { - ignore_unmapped: true, - [agg.getField().name]: params.boundingBox || defaultBoundingBox, - }, - }, - } as any, - { addToAggConfigs: false } - ) - ); - } + if (params.isFilteredByCollar && agg.getField()) { + aggs.push( + agg.aggConfigs.createAggConfig( + { + type: 'filter', + id: 'filter_agg', + enabled: true, + params: { + geo_bounding_box: { + ignore_unmapped: true, + [agg.getField().name]: params.boundingBox || defaultBoundingBox, + }, + }, + } as any, + { addToAggConfigs: false } + ) + ); + } - aggs.push(agg); + aggs.push(agg); - if (params.useGeocentroid) { - aggs.push( - agg.aggConfigs.createAggConfig( - { - type: 'geo_centroid', - enabled: true, - params: { - field: agg.getField(), - }, - }, - { addToAggConfigs: false } - ) - ); - } + if (params.useGeocentroid) { + aggs.push( + agg.aggConfigs.createAggConfig( + { + type: 'geo_centroid', + enabled: true, + params: { + field: agg.getField(), + }, + }, + { addToAggConfigs: false } + ) + ); + } - return aggs; - }, -}); + return aggs; + }, + }, + { getInternalStartServices } + ); diff --git a/src/plugins/data/public/search/aggs/buckets/geo_tile.ts b/src/plugins/data/public/search/aggs/buckets/geo_tile.ts index 759601fc0c180..c981e8400f9a1 100644 --- a/src/plugins/data/public/search/aggs/buckets/geo_tile.ts +++ b/src/plugins/data/public/search/aggs/buckets/geo_tile.ts @@ -20,53 +20,61 @@ import { i18n } from '@kbn/i18n'; import { noop } from 'lodash'; -import { BucketAggType } from './_bucket_agg_type'; +import { BucketAggType, IBucketAggConfig } from './bucket_agg_type'; import { BUCKET_TYPES } from './bucket_agg_types'; import { KBN_FIELD_TYPES } from '../../../../common'; -import { IBucketAggConfig } from './_bucket_agg_type'; import { METRIC_TYPES } from '../metrics/metric_agg_types'; +import { GetInternalStartServicesFn } from '../../../types'; + +export interface GeoTitleBucketAggDependencies { + getInternalStartServices: GetInternalStartServicesFn; +} const geotileGridTitle = i18n.translate('data.search.aggs.buckets.geotileGridTitle', { defaultMessage: 'Geotile', }); -export const geoTileBucketAgg = new BucketAggType({ - name: BUCKET_TYPES.GEOTILE_GRID, - title: geotileGridTitle, - params: [ - { - name: 'field', - type: 'field', - filterFieldTypes: KBN_FIELD_TYPES.GEO_POINT, - }, +export const getGeoTitleBucketAgg = ({ getInternalStartServices }: GeoTitleBucketAggDependencies) => + new BucketAggType( { - name: 'useGeocentroid', - default: true, - write: noop, - }, - { - name: 'precision', - default: 0, - }, - ], - getRequestAggs(agg) { - const aggs = []; - const useGeocentroid = agg.getParam('useGeocentroid'); + name: BUCKET_TYPES.GEOTILE_GRID, + title: geotileGridTitle, + params: [ + { + name: 'field', + type: 'field', + filterFieldTypes: KBN_FIELD_TYPES.GEO_POINT, + }, + { + name: 'useGeocentroid', + default: true, + write: noop, + }, + { + name: 'precision', + default: 0, + }, + ], + getRequestAggs(agg) { + const aggs = []; + const useGeocentroid = agg.getParam('useGeocentroid'); - aggs.push(agg); + aggs.push(agg); - if (useGeocentroid) { - const aggConfig = { - type: METRIC_TYPES.GEO_CENTROID, - enabled: true, - params: { - field: agg.getField(), - }, - }; + if (useGeocentroid) { + const aggConfig = { + type: METRIC_TYPES.GEO_CENTROID, + enabled: true, + params: { + field: agg.getField(), + }, + }; - aggs.push(agg.aggConfigs.createAggConfig(aggConfig, { addToAggConfigs: false })); - } + aggs.push(agg.aggConfigs.createAggConfig(aggConfig, { addToAggConfigs: false })); + } - return aggs as IBucketAggConfig[]; - }, -}); + return aggs as IBucketAggConfig[]; + }, + }, + { getInternalStartServices } + ); diff --git a/src/plugins/data/public/search/aggs/buckets/histogram.test.ts b/src/plugins/data/public/search/aggs/buckets/histogram.test.ts index c61b4ff37935a..bbfc263df4268 100644 --- a/src/plugins/data/public/search/aggs/buckets/histogram.test.ts +++ b/src/plugins/data/public/search/aggs/buckets/histogram.test.ts @@ -17,7 +17,7 @@ * under the License. */ -import { coreMock } from '../../../../../../../src/core/public/mocks'; +import { coreMock, notificationServiceMock } from '../../../../../../../src/core/public/mocks'; import { AggConfigs } from '../agg_configs'; import { mockAggTypesRegistry } from '../test_helpers'; import { BUCKET_TYPES } from './bucket_agg_types'; @@ -27,17 +27,21 @@ import { AutoBounds, HistogramBucketAggDependencies, } from './histogram'; -import { BucketAggType } from './_bucket_agg_type'; +import { BucketAggType } from './bucket_agg_type'; +import { fieldFormatsServiceMock } from '../../../field_formats/mocks'; describe('Histogram Agg', () => { let aggTypesDependencies: HistogramBucketAggDependencies; beforeEach(() => { - const { uiSettings, notifications } = coreMock.createSetup(); + const { uiSettings } = coreMock.createSetup(); aggTypesDependencies = { uiSettings, - notifications, + getInternalStartServices: () => ({ + fieldFormats: fieldFormatsServiceMock.createStartContract(), + notifications: notificationServiceMock.createStartContract(), + }), }; }); @@ -87,18 +91,18 @@ describe('Histogram Agg', () => { histogramType = getHistogramBucketAgg(aggTypesDependencies); }); - it('is ordered', () => { + test('is ordered', () => { expect(histogramType.ordered).toBeDefined(); }); - it('is not ordered by date', () => { + test('is not ordered by date', () => { expect(histogramType.ordered).not.toHaveProperty('date'); }); }); describe('params', () => { describe('intervalBase', () => { - it('should not be written to the DSL', () => { + test('should not be written to the DSL', () => { const aggConfigs = getAggConfigs({ intervalBase: 100, field: { @@ -112,7 +116,7 @@ describe('Histogram Agg', () => { }); describe('interval', () => { - it('accepts a whole number', () => { + test('accepts a whole number', () => { const params = getParams({ interval: 100, }); @@ -120,7 +124,7 @@ describe('Histogram Agg', () => { expect(params).toHaveProperty('interval', 100); }); - it('accepts a decimal number', function() { + test('accepts a decimal number', () => { const params = getParams({ interval: 0.1, }); @@ -128,7 +132,7 @@ describe('Histogram Agg', () => { expect(params).toHaveProperty('interval', 0.1); }); - it('accepts a decimal number string', function() { + test('accepts a decimal number string', () => { const params = getParams({ interval: '0.1', }); @@ -136,7 +140,7 @@ describe('Histogram Agg', () => { expect(params).toHaveProperty('interval', 0.1); }); - it('accepts a whole number string', function() { + test('accepts a whole number string', () => { const params = getParams({ interval: '10', }); @@ -144,7 +148,7 @@ describe('Histogram Agg', () => { expect(params).toHaveProperty('interval', 10); }); - it('fails on non-numeric values', function() { + test('fails on non-numeric values', () => { const params = getParams({ interval: [], }); @@ -181,7 +185,7 @@ describe('Histogram Agg', () => { return aggConfig.write(aggConfigs).params; }; - it('will respect the histogram:maxBars setting', () => { + test('will respect the histogram:maxBars setting', () => { const params = getInterval( 5, { interval: 5 }, @@ -194,19 +198,19 @@ describe('Histogram Agg', () => { expect(params).toHaveProperty('interval', 2000); }); - it('will return specified interval, if bars are below histogram:maxBars config', () => { + test('will return specified interval, if bars are below histogram:maxBars config', () => { const params = getInterval(100, { interval: 5 }); expect(params).toHaveProperty('interval', 5); }); - it('will set to intervalBase if interval is below base', () => { + test('will set to intervalBase if interval is below base', () => { const params = getInterval(1000, { interval: 3, intervalBase: 8 }); expect(params).toHaveProperty('interval', 8); }); - it('will round to nearest intervalBase multiple if interval is above base', () => { + test('will round to nearest intervalBase multiple if interval is above base', () => { const roundUp = getInterval(1000, { interval: 46, intervalBase: 10 }); expect(roundUp).toHaveProperty('interval', 50); @@ -214,13 +218,13 @@ describe('Histogram Agg', () => { expect(roundDown).toHaveProperty('interval', 40); }); - it('will not change interval if it is a multiple of base', () => { + test('will not change interval if it is a multiple of base', () => { const output = getInterval(1000, { interval: 35, intervalBase: 5 }); expect(output).toHaveProperty('interval', 35); }); - it('will round to intervalBase after scaling histogram:maxBars', () => { + test('will round to intervalBase after scaling histogram:maxBars', () => { const output = getInterval(100, { interval: 5, intervalBase: 6 }, { min: 0, max: 1000 }); // 100 buckets in 0 to 1000 would result in an interval of 10, so we should @@ -232,7 +236,7 @@ describe('Histogram Agg', () => { describe('min_doc_count', () => { let output: Record; - it('casts true values to 0', () => { + test('casts true values to 0', () => { output = getParams({ min_doc_count: true }); expect(output).toHaveProperty('min_doc_count', 0); @@ -246,7 +250,7 @@ describe('Histogram Agg', () => { expect(output).toHaveProperty('min_doc_count', 0); }); - it('writes 1 for falsy values', () => { + test('writes 1 for falsy values', () => { output = getParams({ min_doc_count: '' }); expect(output).toHaveProperty('min_doc_count', 1); @@ -258,8 +262,8 @@ describe('Histogram Agg', () => { }); }); - describe('extended_bounds', function() { - it('does not write when only eb.min is set', function() { + describe('extended_bounds', () => { + test('does not write when only eb.min is set', () => { const output = getParams({ has_extended_bounds: true, extended_bounds: { min: 0 }, @@ -267,7 +271,7 @@ describe('Histogram Agg', () => { expect(output).not.toHaveProperty('extended_bounds'); }); - it('does not write when only eb.max is set', function() { + test('does not write when only eb.max is set', () => { const output = getParams({ has_extended_bounds: true, extended_bounds: { max: 0 }, @@ -276,7 +280,7 @@ describe('Histogram Agg', () => { expect(output).not.toHaveProperty('extended_bounds'); }); - it('writes when both eb.min and eb.max are set', function() { + test('writes when both eb.min and eb.max are set', () => { const output = getParams({ has_extended_bounds: true, extended_bounds: { min: 99, max: 100 }, @@ -286,7 +290,7 @@ describe('Histogram Agg', () => { expect(output.extended_bounds).toHaveProperty('max', 100); }); - it('does not write when nothing is set', function() { + test('does not write when nothing is set', () => { const output = getParams({ has_extended_bounds: true, extended_bounds: {}, @@ -295,7 +299,7 @@ describe('Histogram Agg', () => { expect(output).not.toHaveProperty('extended_bounds'); }); - it('does not write when has_extended_bounds is false', function() { + test('does not write when has_extended_bounds is false', () => { const output = getParams({ has_extended_bounds: false, extended_bounds: { min: 99, max: 100 }, diff --git a/src/plugins/data/public/search/aggs/buckets/histogram.ts b/src/plugins/data/public/search/aggs/buckets/histogram.ts index bbffc0912bf0d..f8e8720d24ea9 100644 --- a/src/plugins/data/public/search/aggs/buckets/histogram.ts +++ b/src/plugins/data/public/search/aggs/buckets/histogram.ts @@ -19,12 +19,13 @@ import { get } from 'lodash'; import { i18n } from '@kbn/i18n'; -import { IUiSettingsClient, NotificationsSetup } from 'src/core/public'; +import { IUiSettingsClient } from 'src/core/public'; -import { BucketAggType, IBucketAggConfig } from './_bucket_agg_type'; +import { BucketAggType, IBucketAggConfig } from './bucket_agg_type'; import { createFilterHistogram } from './create_filter/histogram'; import { BUCKET_TYPES } from './bucket_agg_types'; import { KBN_FIELD_TYPES } from '../../../../common'; +import { GetInternalStartServicesFn } from '../../../types'; export interface AutoBounds { min: number; @@ -33,7 +34,7 @@ export interface AutoBounds { export interface HistogramBucketAggDependencies { uiSettings: IUiSettingsClient; - notifications: NotificationsSetup; + getInternalStartServices: GetInternalStartServicesFn; } export interface IBucketHistogramAggConfig extends IBucketAggConfig { @@ -43,164 +44,167 @@ export interface IBucketHistogramAggConfig extends IBucketAggConfig { export const getHistogramBucketAgg = ({ uiSettings, - notifications, + getInternalStartServices, }: HistogramBucketAggDependencies) => - new BucketAggType({ - name: BUCKET_TYPES.HISTOGRAM, - title: i18n.translate('data.search.aggs.buckets.histogramTitle', { - defaultMessage: 'Histogram', - }), - ordered: {}, - makeLabel(aggConfig) { - return aggConfig.getFieldDisplayName(); - }, - createFilter: createFilterHistogram, - decorateAggConfig() { - let autoBounds: AutoBounds; - - return { - setAutoBounds: { - configurable: true, - value(newValue: AutoBounds) { - autoBounds = newValue; + new BucketAggType( + { + name: BUCKET_TYPES.HISTOGRAM, + title: i18n.translate('data.search.aggs.buckets.histogramTitle', { + defaultMessage: 'Histogram', + }), + ordered: {}, + makeLabel(aggConfig) { + return aggConfig.getFieldDisplayName(); + }, + createFilter: createFilterHistogram, + decorateAggConfig() { + let autoBounds: AutoBounds; + + return { + setAutoBounds: { + configurable: true, + value(newValue: AutoBounds) { + autoBounds = newValue; + }, }, - }, - getAutoBounds: { - configurable: true, - value() { - return autoBounds; + getAutoBounds: { + configurable: true, + value() { + return autoBounds; + }, }, - }, - }; - }, - params: [ - { - name: 'field', - type: 'field', - filterFieldTypes: KBN_FIELD_TYPES.NUMBER, - }, - { - /* - * This parameter can be set if you want the auto scaled interval to always - * be a multiple of a specific base. - */ - name: 'intervalBase', - default: null, - write: () => {}, + }; }, - { - name: 'interval', - modifyAggConfigOnSearchRequestStart( - aggConfig: IBucketHistogramAggConfig, - searchSource: any, - options: any - ) { - const field = aggConfig.getField(); - const aggBody = field.scripted - ? { script: { source: field.script, lang: field.lang } } - : { field: field.name }; - - const childSearchSource = searchSource - .createChild() - .setField('size', 0) - .setField('aggs', { - maxAgg: { - max: aggBody, - }, - minAgg: { - min: aggBody, - }, - }); - - return childSearchSource - .fetch(options) - .then((resp: any) => { - aggConfig.setAutoBounds({ - min: get(resp, 'aggregations.minAgg.value'), - max: get(resp, 'aggregations.maxAgg.value'), - }); - }) - .catch((e: Error) => { - if (e.name === 'AbortError') return; - notifications.toasts.addWarning( - i18n.translate('data.search.aggs.histogram.missingMaxMinValuesWarning', { - defaultMessage: - 'Unable to retrieve max and min values to auto-scale histogram buckets. This may lead to poor visualization performance.', - }) - ); - }); + params: [ + { + name: 'field', + type: 'field', + filterFieldTypes: KBN_FIELD_TYPES.NUMBER, + }, + { + /* + * This parameter can be set if you want the auto scaled interval to always + * be a multiple of a specific base. + */ + name: 'intervalBase', + default: null, + write: () => {}, }, - write(aggConfig, output) { - let interval = parseFloat(aggConfig.params.interval); - if (interval <= 0) { - interval = 1; - } - const autoBounds = aggConfig.getAutoBounds(); - - // ensure interval does not create too many buckets and crash browser - if (autoBounds) { - const range = autoBounds.max - autoBounds.min; - const bars = range / interval; - - if (bars > uiSettings.get('histogram:maxBars')) { - const minInterval = range / uiSettings.get('histogram:maxBars'); - - // Round interval by order of magnitude to provide clean intervals - // Always round interval up so there will always be less buckets than histogram:maxBars - const orderOfMagnitude = Math.pow(10, Math.floor(Math.log10(minInterval))); - let roundInterval = orderOfMagnitude; - - while (roundInterval < minInterval) { - roundInterval += orderOfMagnitude; + { + name: 'interval', + modifyAggConfigOnSearchRequestStart( + aggConfig: IBucketHistogramAggConfig, + searchSource: any, + options: any + ) { + const field = aggConfig.getField(); + const aggBody = field.scripted + ? { script: { source: field.script, lang: field.lang } } + : { field: field.name }; + + const childSearchSource = searchSource + .createChild() + .setField('size', 0) + .setField('aggs', { + maxAgg: { + max: aggBody, + }, + minAgg: { + min: aggBody, + }, + }); + + return childSearchSource + .fetch(options) + .then((resp: any) => { + aggConfig.setAutoBounds({ + min: get(resp, 'aggregations.minAgg.value'), + max: get(resp, 'aggregations.maxAgg.value'), + }); + }) + .catch((e: Error) => { + if (e.name === 'AbortError') return; + getInternalStartServices().notifications.toasts.addWarning( + i18n.translate('data.search.aggs.histogram.missingMaxMinValuesWarning', { + defaultMessage: + 'Unable to retrieve max and min values to auto-scale histogram buckets. This may lead to poor visualization performance.', + }) + ); + }); + }, + write(aggConfig, output) { + let interval = parseFloat(aggConfig.params.interval); + if (interval <= 0) { + interval = 1; + } + const autoBounds = aggConfig.getAutoBounds(); + + // ensure interval does not create too many buckets and crash browser + if (autoBounds) { + const range = autoBounds.max - autoBounds.min; + const bars = range / interval; + + if (bars > uiSettings.get('histogram:maxBars')) { + const minInterval = range / uiSettings.get('histogram:maxBars'); + + // Round interval by order of magnitude to provide clean intervals + // Always round interval up so there will always be less buckets than histogram:maxBars + const orderOfMagnitude = Math.pow(10, Math.floor(Math.log10(minInterval))); + let roundInterval = orderOfMagnitude; + + while (roundInterval < minInterval) { + roundInterval += orderOfMagnitude; + } + interval = roundInterval; } - interval = roundInterval; } - } - const base = aggConfig.params.intervalBase; - - if (base) { - if (interval < base) { - // In case the specified interval is below the base, just increase it to it's base - interval = base; - } else if (interval % base !== 0) { - // In case the interval is not a multiple of the base round it to the next base - interval = Math.round(interval / base) * base; + const base = aggConfig.params.intervalBase; + + if (base) { + if (interval < base) { + // In case the specified interval is below the base, just increase it to it's base + interval = base; + } else if (interval % base !== 0) { + // In case the interval is not a multiple of the base round it to the next base + interval = Math.round(interval / base) * base; + } } - } - output.params.interval = interval; + output.params.interval = interval; + }, }, - }, - { - name: 'min_doc_count', - default: false, - write(aggConfig, output) { - if (aggConfig.params.min_doc_count) { - output.params.min_doc_count = 0; - } else { - output.params.min_doc_count = 1; - } + { + name: 'min_doc_count', + default: false, + write(aggConfig, output) { + if (aggConfig.params.min_doc_count) { + output.params.min_doc_count = 0; + } else { + output.params.min_doc_count = 1; + } + }, }, - }, - { - name: 'has_extended_bounds', - default: false, - write: () => {}, - }, - { - name: 'extended_bounds', - default: { - min: '', - max: '', + { + name: 'has_extended_bounds', + default: false, + write: () => {}, }, - write(aggConfig, output) { - const { min, max } = aggConfig.params.extended_bounds; + { + name: 'extended_bounds', + default: { + min: '', + max: '', + }, + write(aggConfig, output) { + const { min, max } = aggConfig.params.extended_bounds; - if (aggConfig.params.has_extended_bounds && (min || min === 0) && (max || max === 0)) { - output.params.extended_bounds = { min, max }; - } + if (aggConfig.params.has_extended_bounds && (min || min === 0) && (max || max === 0)) { + output.params.extended_bounds = { min, max }; + } + }, + shouldShow: (aggConfig: IBucketAggConfig) => aggConfig.params.has_extended_bounds, }, - shouldShow: (aggConfig: IBucketAggConfig) => aggConfig.params.has_extended_bounds, - }, - ], - }); + ], + }, + { getInternalStartServices } + ); diff --git a/src/plugins/data/public/search/aggs/buckets/ip_range.ts b/src/plugins/data/public/search/aggs/buckets/ip_range.ts index da6866d40a52f..bde347d6e673d 100644 --- a/src/plugins/data/public/search/aggs/buckets/ip_range.ts +++ b/src/plugins/data/public/search/aggs/buckets/ip_range.ts @@ -19,77 +19,85 @@ import { noop, map, omit, isNull } from 'lodash'; import { i18n } from '@kbn/i18n'; -import { BucketAggType } from './_bucket_agg_type'; +import { BucketAggType } from './bucket_agg_type'; import { BUCKET_TYPES } from './bucket_agg_types'; import { createFilterIpRange } from './create_filter/ip_range'; import { IpRangeKey, convertIPRangeToString } from './lib/ip_range'; import { KBN_FIELD_TYPES, FieldFormat, TEXT_CONTEXT_TYPE } from '../../../../common'; -import { getFieldFormats } from '../../../../public/services'; +import { GetInternalStartServicesFn } from '../../../types'; const ipRangeTitle = i18n.translate('data.search.aggs.buckets.ipRangeTitle', { defaultMessage: 'IPv4 Range', }); -export const ipRangeBucketAgg = new BucketAggType({ - name: BUCKET_TYPES.IP_RANGE, - title: ipRangeTitle, - createFilter: createFilterIpRange, - getKey(bucket, key, agg): IpRangeKey { - if (agg.params.ipRangeType === 'mask') { - return { type: 'mask', mask: key }; - } - return { type: 'range', from: bucket.from, to: bucket.to }; - }, - getFormat(agg) { - const fieldFormatsService = getFieldFormats(); - const formatter = agg.fieldOwnFormatter( - TEXT_CONTEXT_TYPE, - fieldFormatsService.getDefaultInstance(KBN_FIELD_TYPES.IP) - ); - const IpRangeFormat = FieldFormat.from(function(range: IpRangeKey) { - return convertIPRangeToString(range, formatter); - }); - return new IpRangeFormat(); - }, - makeLabel(aggConfig) { - return i18n.translate('data.search.aggs.buckets.ipRangeLabel', { - defaultMessage: '{fieldName} IP ranges', - values: { - fieldName: aggConfig.getFieldDisplayName(), - }, - }); - }, - params: [ - { - name: 'field', - type: 'field', - filterFieldTypes: KBN_FIELD_TYPES.IP, - }, - { - name: 'ipRangeType', - default: 'fromTo', - write: noop, - }, +export interface IpRangeBucketAggDependencies { + getInternalStartServices: GetInternalStartServicesFn; +} + +export const getIpRangeBucketAgg = ({ getInternalStartServices }: IpRangeBucketAggDependencies) => + new BucketAggType( { - name: 'ranges', - default: { - fromTo: [ - { from: '0.0.0.0', to: '127.255.255.255' }, - { from: '128.0.0.0', to: '191.255.255.255' }, - ], - mask: [{ mask: '0.0.0.0/1' }, { mask: '128.0.0.0/2' }], + name: BUCKET_TYPES.IP_RANGE, + title: ipRangeTitle, + createFilter: createFilterIpRange, + getKey(bucket, key, agg): IpRangeKey { + if (agg.params.ipRangeType === 'mask') { + return { type: 'mask', mask: key }; + } + return { type: 'range', from: bucket.from, to: bucket.to }; + }, + getFormat(agg) { + const { fieldFormats } = getInternalStartServices(); + const formatter = agg.fieldOwnFormatter( + TEXT_CONTEXT_TYPE, + fieldFormats.getDefaultInstance(KBN_FIELD_TYPES.IP) + ); + const IpRangeFormat = FieldFormat.from(function(range: IpRangeKey) { + return convertIPRangeToString(range, formatter); + }); + return new IpRangeFormat(); + }, + makeLabel(aggConfig) { + return i18n.translate('data.search.aggs.buckets.ipRangeLabel', { + defaultMessage: '{fieldName} IP ranges', + values: { + fieldName: aggConfig.getFieldDisplayName(), + }, + }); }, - write(aggConfig, output) { - const ipRangeType = aggConfig.params.ipRangeType; - let ranges = aggConfig.params.ranges[ipRangeType]; + params: [ + { + name: 'field', + type: 'field', + filterFieldTypes: KBN_FIELD_TYPES.IP, + }, + { + name: 'ipRangeType', + default: 'fromTo', + write: noop, + }, + { + name: 'ranges', + default: { + fromTo: [ + { from: '0.0.0.0', to: '127.255.255.255' }, + { from: '128.0.0.0', to: '191.255.255.255' }, + ], + mask: [{ mask: '0.0.0.0/1' }, { mask: '128.0.0.0/2' }], + }, + write(aggConfig, output) { + const ipRangeType = aggConfig.params.ipRangeType; + let ranges = aggConfig.params.ranges[ipRangeType]; - if (ipRangeType === 'fromTo') { - ranges = map(ranges, (range: any) => omit(range, isNull)); - } + if (ipRangeType === 'fromTo') { + ranges = map(ranges, (range: any) => omit(range, isNull)); + } - output.params.ranges = ranges; - }, + output.params.ranges = ranges; + }, + }, + ], }, - ], -}); + { getInternalStartServices } + ); diff --git a/src/plugins/data/public/search/aggs/buckets/migrate_include_exclude_format.ts b/src/plugins/data/public/search/aggs/buckets/migrate_include_exclude_format.ts index d94477b588f8d..0beeb1c372275 100644 --- a/src/plugins/data/public/search/aggs/buckets/migrate_include_exclude_format.ts +++ b/src/plugins/data/public/search/aggs/buckets/migrate_include_exclude_format.ts @@ -18,7 +18,7 @@ */ import { isString, isObject } from 'lodash'; -import { IBucketAggConfig, BucketAggType, BucketAggParam } from './_bucket_agg_type'; +import { IBucketAggConfig, BucketAggType, BucketAggParam } from './bucket_agg_type'; import { IAggConfig } from '../agg_config'; export const isType = (type: string) => { diff --git a/src/plugins/data/public/search/aggs/buckets/range.test.ts b/src/plugins/data/public/search/aggs/buckets/range.test.ts index bf3711543ae88..a1f0ab6a2326a 100644 --- a/src/plugins/data/public/search/aggs/buckets/range.test.ts +++ b/src/plugins/data/public/search/aggs/buckets/range.test.ts @@ -17,11 +17,13 @@ * under the License. */ -import { rangeBucketAgg } from './range'; +import { getRangeBucketAgg, RangeBucketAggDependencies } from './range'; import { AggConfigs } from '../agg_configs'; import { mockDataServices, mockAggTypesRegistry } from '../test_helpers'; import { BUCKET_TYPES } from './bucket_agg_types'; import { FieldFormatsGetConfigFn, NumberFormat } from '../../../../common'; +import { fieldFormatsServiceMock } from '../../../field_formats/mocks'; +import { notificationServiceMock } from '../../../../../../../src/core/public/mocks'; const buckets = [ { @@ -44,7 +46,16 @@ const buckets = [ ]; describe('Range Agg', () => { + let aggTypesDependencies: RangeBucketAggDependencies; + beforeEach(() => { + aggTypesDependencies = { + getInternalStartServices: () => ({ + fieldFormats: fieldFormatsServiceMock.createStartContract(), + notifications: notificationServiceMock.createStartContract(), + }), + }; + mockDataServices(); }); @@ -84,15 +95,14 @@ describe('Range Agg', () => { }, }, ], - { typesRegistry: mockAggTypesRegistry([rangeBucketAgg]) } + { typesRegistry: mockAggTypesRegistry([getRangeBucketAgg(aggTypesDependencies)]) } ); }; describe('formating', () => { - it('formats bucket keys properly', () => { + test('formats bucket keys properly', () => { const aggConfigs = getAggConfigs(); const agg = aggConfigs.aggs[0]; - const format = (val: any) => agg.fieldFormatter()(agg.getKey(val)); expect(format(buckets[0])).toBe('≥ -∞ and < 1 KB'); diff --git a/src/plugins/data/public/search/aggs/buckets/range.ts b/src/plugins/data/public/search/aggs/buckets/range.ts index 036a0d4c1e8da..2c1303814a88a 100644 --- a/src/plugins/data/public/search/aggs/buckets/range.ts +++ b/src/plugins/data/public/search/aggs/buckets/range.ts @@ -18,11 +18,12 @@ */ import { i18n } from '@kbn/i18n'; -import { BucketAggType } from './_bucket_agg_type'; +import { BucketAggType } from './bucket_agg_type'; import { FieldFormat, KBN_FIELD_TYPES } from '../../../../common'; import { RangeKey } from './range_key'; import { createFilterRange } from './create_filter/range'; import { BUCKET_TYPES } from './bucket_agg_types'; +import { GetInternalStartServicesFn } from '../../../types'; const keyCaches = new WeakMap(); const formats = new WeakMap(); @@ -31,76 +32,84 @@ const rangeTitle = i18n.translate('data.search.aggs.buckets.rangeTitle', { defaultMessage: 'Range', }); -export const rangeBucketAgg = new BucketAggType({ - name: BUCKET_TYPES.RANGE, - title: rangeTitle, - createFilter: createFilterRange, - makeLabel(aggConfig) { - return i18n.translate('data.search.aggs.aggTypesLabel', { - defaultMessage: '{fieldName} ranges', - values: { - fieldName: aggConfig.getFieldDisplayName(), +export interface RangeBucketAggDependencies { + getInternalStartServices: GetInternalStartServicesFn; +} + +export const getRangeBucketAgg = ({ getInternalStartServices }: RangeBucketAggDependencies) => + new BucketAggType( + { + name: BUCKET_TYPES.RANGE, + title: rangeTitle, + createFilter: createFilterRange, + makeLabel(aggConfig) { + return i18n.translate('data.search.aggs.aggTypesLabel', { + defaultMessage: '{fieldName} ranges', + values: { + fieldName: aggConfig.getFieldDisplayName(), + }, + }); }, - }); - }, - getKey(bucket, key, agg) { - let keys = keyCaches.get(agg); + getKey(bucket, key, agg) { + let keys = keyCaches.get(agg); - if (!keys) { - keys = new Map(); - keyCaches.set(agg, keys); - } + if (!keys) { + keys = new Map(); + keyCaches.set(agg, keys); + } - const id = RangeKey.idBucket(bucket); + const id = RangeKey.idBucket(bucket); - key = keys.get(id); - if (!key) { - key = new RangeKey(bucket); - keys.set(id, key); - } + key = keys.get(id); + if (!key) { + key = new RangeKey(bucket); + keys.set(id, key); + } - return key; - }, - getFormat(agg) { - let aggFormat = formats.get(agg); - if (aggFormat) return aggFormat; + return key; + }, + getFormat(agg) { + let aggFormat = formats.get(agg); + if (aggFormat) return aggFormat; - const RangeFormat = FieldFormat.from((range: any) => { - const format = agg.fieldOwnFormatter(); - const gte = '\u2265'; - const lt = '\u003c'; - return i18n.translate('data.search.aggs.aggTypes.rangesFormatMessage', { - defaultMessage: '{gte} {from} and {lt} {to}', - values: { - gte, - from: format(range.gte), - lt, - to: format(range.lt), - }, - }); - }); + const RangeFormat = FieldFormat.from((range: any) => { + const format = agg.fieldOwnFormatter(); + const gte = '\u2265'; + const lt = '\u003c'; + return i18n.translate('data.search.aggs.aggTypes.rangesFormatMessage', { + defaultMessage: '{gte} {from} and {lt} {to}', + values: { + gte, + from: format(range.gte), + lt, + to: format(range.lt), + }, + }); + }); - aggFormat = new RangeFormat(); + aggFormat = new RangeFormat(); - formats.set(agg, aggFormat); - return aggFormat; - }, - params: [ - { - name: 'field', - type: 'field', - filterFieldTypes: [KBN_FIELD_TYPES.NUMBER], - }, - { - name: 'ranges', - default: [ - { from: 0, to: 1000 }, - { from: 1000, to: 2000 }, - ], - write(aggConfig, output) { - output.params.ranges = aggConfig.params.ranges; - output.params.keyed = true; + formats.set(agg, aggFormat); + return aggFormat; }, + params: [ + { + name: 'field', + type: 'field', + filterFieldTypes: [KBN_FIELD_TYPES.NUMBER], + }, + { + name: 'ranges', + default: [ + { from: 0, to: 1000 }, + { from: 1000, to: 2000 }, + ], + write(aggConfig, output) { + output.params.ranges = aggConfig.params.ranges; + output.params.keyed = true; + }, + }, + ], }, - ], -}); + { getInternalStartServices } + ); diff --git a/src/plugins/data/public/search/aggs/buckets/significant_terms.test.ts b/src/plugins/data/public/search/aggs/buckets/significant_terms.test.ts index 1c221126c35e0..761d0ced6a114 100644 --- a/src/plugins/data/public/search/aggs/buckets/significant_terms.test.ts +++ b/src/plugins/data/public/search/aggs/buckets/significant_terms.test.ts @@ -20,12 +20,27 @@ import { AggConfigs, IAggConfigs } from '../agg_configs'; import { mockAggTypesRegistry } from '../test_helpers'; import { BUCKET_TYPES } from './bucket_agg_types'; -import { significantTermsBucketAgg } from './significant_terms'; -import { IBucketAggConfig } from './_bucket_agg_type'; +import { + getSignificantTermsBucketAgg, + SignificantTermsBucketAggDependencies, +} from './significant_terms'; +import { fieldFormatsServiceMock } from '../../../field_formats/mocks'; +import { notificationServiceMock } from '../../../../../../../src/core/public/mocks'; describe('Significant Terms Agg', () => { describe('order agg editor UI', () => { describe('convert include/exclude from old format', () => { + let aggTypesDependencies: SignificantTermsBucketAggDependencies; + + beforeEach(() => { + aggTypesDependencies = { + getInternalStartServices: () => ({ + fieldFormats: fieldFormatsServiceMock.createStartContract(), + notifications: notificationServiceMock.createStartContract(), + }), + }; + }); + const getAggConfigs = (params: Record = {}) => { const indexPattern = { id: '1234', @@ -51,7 +66,11 @@ describe('Significant Terms Agg', () => { params, }, ], - { typesRegistry: mockAggTypesRegistry([significantTermsBucketAgg]) } + { + typesRegistry: mockAggTypesRegistry([ + getSignificantTermsBucketAgg(aggTypesDependencies), + ]), + } ); }; @@ -64,19 +83,19 @@ describe('Significant Terms Agg', () => { expect(params.exclude).toBe('400'); }; - it('should generate correct label', () => { + test('should generate correct label', () => { const aggConfigs = getAggConfigs({ size: 'SIZE', field: { name: 'FIELD', }, }); - const label = significantTermsBucketAgg.makeLabel(aggConfigs.aggs[0] as IBucketAggConfig); + const label = aggConfigs.aggs[0].makeLabel(); expect(label).toBe('Top SIZE unusual terms in FIELD'); }); - it('should doesnt do anything with string type', () => { + test('should doesnt do anything with string type', () => { const aggConfigs = getAggConfigs({ include: '404', exclude: '400', @@ -89,7 +108,7 @@ describe('Significant Terms Agg', () => { testSerializeAndWrite(aggConfigs); }); - it('should converts object to string type', () => { + test('should converts object to string type', () => { const aggConfigs = getAggConfigs({ include: { pattern: '404', diff --git a/src/plugins/data/public/search/aggs/buckets/significant_terms.ts b/src/plugins/data/public/search/aggs/buckets/significant_terms.ts index f12ebe58e2de2..49d797f3afbc9 100644 --- a/src/plugins/data/public/search/aggs/buckets/significant_terms.ts +++ b/src/plugins/data/public/search/aggs/buckets/significant_terms.ts @@ -18,59 +18,72 @@ */ import { i18n } from '@kbn/i18n'; -import { BucketAggType } from './_bucket_agg_type'; +import { BucketAggType } from './bucket_agg_type'; import { createFilterTerms } from './create_filter/terms'; import { isStringType, migrateIncludeExcludeFormat } from './migrate_include_exclude_format'; import { BUCKET_TYPES } from './bucket_agg_types'; import { KBN_FIELD_TYPES } from '../../../../common'; +import { GetInternalStartServicesFn } from '../../../types'; const significantTermsTitle = i18n.translate('data.search.aggs.buckets.significantTermsTitle', { defaultMessage: 'Significant Terms', }); -export const significantTermsBucketAgg = new BucketAggType({ - name: BUCKET_TYPES.SIGNIFICANT_TERMS, - title: significantTermsTitle, - makeLabel(aggConfig) { - return i18n.translate('data.search.aggs.buckets.significantTermsLabel', { - defaultMessage: 'Top {size} unusual terms in {fieldName}', - values: { - size: aggConfig.params.size, - fieldName: aggConfig.getFieldDisplayName(), - }, - }); - }, - createFilter: createFilterTerms, - params: [ - { - name: 'field', - type: 'field', - scriptable: false, - filterFieldTypes: KBN_FIELD_TYPES.STRING, - }, - { - name: 'size', - default: '', - }, +export interface SignificantTermsBucketAggDependencies { + getInternalStartServices: GetInternalStartServicesFn; +} + +export const getSignificantTermsBucketAgg = ({ + getInternalStartServices, +}: SignificantTermsBucketAggDependencies) => + new BucketAggType( { - name: 'exclude', - displayName: i18n.translate('data.search.aggs.buckets.significantTerms.excludeLabel', { - defaultMessage: 'Exclude', - }), - type: 'string', - advanced: true, - shouldShow: isStringType, - ...migrateIncludeExcludeFormat, + name: BUCKET_TYPES.SIGNIFICANT_TERMS, + title: significantTermsTitle, + makeLabel(aggConfig) { + return i18n.translate('data.search.aggs.buckets.significantTermsLabel', { + defaultMessage: 'Top {size} unusual terms in {fieldName}', + values: { + size: aggConfig.params.size, + fieldName: aggConfig.getFieldDisplayName(), + }, + }); + }, + createFilter: createFilterTerms, + params: [ + { + name: 'field', + type: 'field', + scriptable: false, + filterFieldTypes: KBN_FIELD_TYPES.STRING, + }, + { + name: 'size', + default: '', + }, + { + name: 'exclude', + displayName: i18n.translate('data.search.aggs.buckets.significantTerms.excludeLabel', { + defaultMessage: 'Exclude', + }), + type: 'string', + advanced: true, + shouldShow: isStringType, + ...migrateIncludeExcludeFormat, + }, + { + name: 'include', + displayName: i18n.translate('data.search.aggs.buckets.significantTerms.includeLabel', { + defaultMessage: 'Include', + }), + type: 'string', + advanced: true, + shouldShow: isStringType, + ...migrateIncludeExcludeFormat, + }, + ], }, { - name: 'include', - displayName: i18n.translate('data.search.aggs.buckets.significantTerms.includeLabel', { - defaultMessage: 'Include', - }), - type: 'string', - advanced: true, - shouldShow: isStringType, - ...migrateIncludeExcludeFormat, - }, - ], -}); + getInternalStartServices, + } + ); diff --git a/src/plugins/data/public/search/aggs/buckets/terms.test.ts b/src/plugins/data/public/search/aggs/buckets/terms.test.ts index 280d78f6620bd..5afe7d0b0c35c 100644 --- a/src/plugins/data/public/search/aggs/buckets/terms.test.ts +++ b/src/plugins/data/public/search/aggs/buckets/terms.test.ts @@ -51,7 +51,7 @@ describe('Terms Agg', () => { ); }; - it('converts object to string type', function() { + test('converts object to string type', () => { const aggConfigs = getAggConfigs({ include: { pattern: '404', diff --git a/src/plugins/data/public/search/aggs/buckets/terms.ts b/src/plugins/data/public/search/aggs/buckets/terms.ts index 813c657934a76..5baa38af0e8d6 100644 --- a/src/plugins/data/public/search/aggs/buckets/terms.ts +++ b/src/plugins/data/public/search/aggs/buckets/terms.ts @@ -19,9 +19,8 @@ import { noop } from 'lodash'; import { i18n } from '@kbn/i18n'; -import { BucketAggType } from './_bucket_agg_type'; +import { BucketAggType, IBucketAggConfig } from './bucket_agg_type'; import { BUCKET_TYPES } from './bucket_agg_types'; -import { IBucketAggConfig } from './_bucket_agg_type'; import { createFilterTerms } from './create_filter/terms'; import { isStringType, migrateIncludeExcludeFormat } from './migrate_include_exclude_format'; import { IAggConfigs } from '../agg_configs'; @@ -36,6 +35,7 @@ import { mergeOtherBucketAggResponse, updateMissingBucket, } from './_terms_other_bucket_helper'; +import { GetInternalStartServicesFn } from '../../../types'; export const termsAggFilter = [ '!top_hits', @@ -56,220 +56,230 @@ const termsTitle = i18n.translate('data.search.aggs.buckets.termsTitle', { defaultMessage: 'Terms', }); -export const termsBucketAgg = new BucketAggType({ - name: BUCKET_TYPES.TERMS, - title: termsTitle, - makeLabel(agg) { - const params = agg.params; - return agg.getFieldDisplayName() + ': ' + params.order.text; - }, - getFormat(bucket): IFieldFormat { - return { - getConverterFor: (type: FieldFormatsContentType) => { - return (val: any) => { - if (val === '__other__') { - return bucket.params.otherBucketLabel; - } - if (val === '__missing__') { - return bucket.params.missingBucketLabel; - } +export interface TermsBucketAggDependencies { + getInternalStartServices: GetInternalStartServicesFn; +} - return bucket.params.field.format.convert(val, type); - }; +export const getTermsBucketAgg = ({ getInternalStartServices }: TermsBucketAggDependencies) => + new BucketAggType( + { + name: BUCKET_TYPES.TERMS, + title: termsTitle, + makeLabel(agg) { + const params = agg.params; + return agg.getFieldDisplayName() + ': ' + params.order.text; }, - } as IFieldFormat; - }, - createFilter: createFilterTerms, - postFlightRequest: async ( - resp: any, - aggConfigs: IAggConfigs, - aggConfig: IBucketAggConfig, - searchSource: ISearchSource, - inspectorAdapters: Adapters, - abortSignal?: AbortSignal - ) => { - if (!resp.aggregations) return resp; - const nestedSearchSource = searchSource.createChild(); - if (aggConfig.params.otherBucket) { - const filterAgg = buildOtherBucketAgg(aggConfigs, aggConfig, resp); - if (!filterAgg) return resp; + getFormat(bucket): IFieldFormat { + return { + getConverterFor: (type: FieldFormatsContentType) => { + return (val: any) => { + if (val === '__other__') { + return bucket.params.otherBucketLabel; + } + if (val === '__missing__') { + return bucket.params.missingBucketLabel; + } - nestedSearchSource.setField('aggs', filterAgg); + return bucket.params.field.format.convert(val, type); + }; + }, + } as IFieldFormat; + }, + createFilter: createFilterTerms, + postFlightRequest: async ( + resp: any, + aggConfigs: IAggConfigs, + aggConfig: IBucketAggConfig, + searchSource: ISearchSource, + inspectorAdapters: Adapters, + abortSignal?: AbortSignal + ) => { + if (!resp.aggregations) return resp; + const nestedSearchSource = searchSource.createChild(); + if (aggConfig.params.otherBucket) { + const filterAgg = buildOtherBucketAgg(aggConfigs, aggConfig, resp); + if (!filterAgg) return resp; - const request = inspectorAdapters.requests.start( - i18n.translate('data.search.aggs.buckets.terms.otherBucketTitle', { - defaultMessage: 'Other bucket', - }), - { - description: i18n.translate('data.search.aggs.buckets.terms.otherBucketDescription', { - defaultMessage: - 'This request counts the number of documents that fall ' + - 'outside the criterion of the data buckets.', - }), - } - ); - nestedSearchSource.getSearchRequestBody().then((body: string) => { - request.json(body); - }); - request.stats(getRequestInspectorStats(nestedSearchSource)); + nestedSearchSource.setField('aggs', filterAgg); - const response = await nestedSearchSource.fetch({ abortSignal }); - request.stats(getResponseInspectorStats(nestedSearchSource, response)).ok({ json: response }); - resp = mergeOtherBucketAggResponse(aggConfigs, resp, response, aggConfig, filterAgg()); - } - if (aggConfig.params.missingBucket) { - resp = updateMissingBucket(resp, aggConfigs, aggConfig); - } - return resp; - }, - params: [ - { - name: 'field', - type: 'field', - filterFieldTypes: [ - KBN_FIELD_TYPES.NUMBER, - KBN_FIELD_TYPES.BOOLEAN, - KBN_FIELD_TYPES.DATE, - KBN_FIELD_TYPES.IP, - KBN_FIELD_TYPES.STRING, - ], - }, - { - name: 'orderBy', - write: noop, // prevent default write, it's handled by orderAgg - }, - { - name: 'orderAgg', - type: 'agg', - allowedAggs: termsAggFilter, - default: null, - makeAgg(termsAgg, state) { - state = state || {}; - state.schema = 'orderAgg'; - const orderAgg = termsAgg.aggConfigs.createAggConfig(state, { - addToAggConfigs: false, - }); - orderAgg.id = termsAgg.id + '-orderAgg'; + const request = inspectorAdapters.requests.start( + i18n.translate('data.search.aggs.buckets.terms.otherBucketTitle', { + defaultMessage: 'Other bucket', + }), + { + description: i18n.translate('data.search.aggs.buckets.terms.otherBucketDescription', { + defaultMessage: + 'This request counts the number of documents that fall ' + + 'outside the criterion of the data buckets.', + }), + } + ); + nestedSearchSource.getSearchRequestBody().then((body: string) => { + request.json(body); + }); + request.stats(getRequestInspectorStats(nestedSearchSource)); - return orderAgg; + const response = await nestedSearchSource.fetch({ abortSignal }); + request + .stats(getResponseInspectorStats(nestedSearchSource, response)) + .ok({ json: response }); + resp = mergeOtherBucketAggResponse(aggConfigs, resp, response, aggConfig, filterAgg()); + } + if (aggConfig.params.missingBucket) { + resp = updateMissingBucket(resp, aggConfigs, aggConfig); + } + return resp; }, - write(agg, output, aggs) { - const dir = agg.params.order.value; - const order: Record = (output.params.order = {}); + params: [ + { + name: 'field', + type: 'field', + filterFieldTypes: [ + KBN_FIELD_TYPES.NUMBER, + KBN_FIELD_TYPES.BOOLEAN, + KBN_FIELD_TYPES.DATE, + KBN_FIELD_TYPES.IP, + KBN_FIELD_TYPES.STRING, + ], + }, + { + name: 'orderBy', + write: noop, // prevent default write, it's handled by orderAgg + }, + { + name: 'orderAgg', + type: 'agg', + allowedAggs: termsAggFilter, + default: null, + makeAgg(termsAgg, state) { + state = state || {}; + state.schema = 'orderAgg'; + const orderAgg = termsAgg.aggConfigs.createAggConfig(state, { + addToAggConfigs: false, + }); + orderAgg.id = termsAgg.id + '-orderAgg'; - let orderAgg = agg.params.orderAgg || aggs!.getResponseAggById(agg.params.orderBy); + return orderAgg; + }, + write(agg, output, aggs) { + const dir = agg.params.order.value; + const order: Record = (output.params.order = {}); - // TODO: This works around an Elasticsearch bug the always casts terms agg scripts to strings - // thus causing issues with filtering. This probably causes other issues since float might not - // be able to contain the number on the elasticsearch side - if (output.params.script) { - output.params.value_type = - agg.getField().type === 'number' ? 'float' : agg.getField().type; - } + let orderAgg = agg.params.orderAgg || aggs!.getResponseAggById(agg.params.orderBy); - if (agg.params.missingBucket && agg.params.field.type === 'string') { - output.params.missing = '__missing__'; - } + // TODO: This works around an Elasticsearch bug the always casts terms agg scripts to strings + // thus causing issues with filtering. This probably causes other issues since float might not + // be able to contain the number on the elasticsearch side + if (output.params.script) { + output.params.value_type = + agg.getField().type === 'number' ? 'float' : agg.getField().type; + } - if (!orderAgg) { - order[agg.params.orderBy || '_count'] = dir; - return; - } + if (agg.params.missingBucket && agg.params.field.type === 'string') { + output.params.missing = '__missing__'; + } - if (orderAgg.type.name === 'count') { - order._count = dir; - return; - } + if (!orderAgg) { + order[agg.params.orderBy || '_count'] = dir; + return; + } - const orderAggId = orderAgg.id; + if (orderAgg.type.name === 'count') { + order._count = dir; + return; + } - if (orderAgg.parentId && aggs) { - orderAgg = aggs.byId(orderAgg.parentId); - } + const orderAggId = orderAgg.id; - output.subAggs = (output.subAggs || []).concat(orderAgg); - order[orderAggId] = dir; - }, - }, - { - name: 'order', - type: 'optioned', - default: 'desc', - options: [ + if (orderAgg.parentId && aggs) { + orderAgg = aggs.byId(orderAgg.parentId); + } + + output.subAggs = (output.subAggs || []).concat(orderAgg); + order[orderAggId] = dir; + }, + }, { - text: i18n.translate('data.search.aggs.buckets.terms.orderDescendingTitle', { - defaultMessage: 'Descending', + name: 'order', + type: 'optioned', + default: 'desc', + options: [ + { + text: i18n.translate('data.search.aggs.buckets.terms.orderDescendingTitle', { + defaultMessage: 'Descending', + }), + value: 'desc', + }, + { + text: i18n.translate('data.search.aggs.buckets.terms.orderAscendingTitle', { + defaultMessage: 'Ascending', + }), + value: 'asc', + }, + ], + write: noop, // prevent default write, it's handled by orderAgg + }, + { + name: 'size', + default: 5, + }, + { + name: 'otherBucket', + default: false, + write: noop, + }, + { + name: 'otherBucketLabel', + type: 'string', + default: i18n.translate('data.search.aggs.buckets.terms.otherBucketLabel', { + defaultMessage: 'Other', + }), + displayName: i18n.translate('data.search.aggs.otherBucket.labelForOtherBucketLabel', { + defaultMessage: 'Label for other bucket', }), - value: 'desc', + shouldShow: agg => agg.getParam('otherBucket'), + write: noop, }, { - text: i18n.translate('data.search.aggs.buckets.terms.orderAscendingTitle', { - defaultMessage: 'Ascending', + name: 'missingBucket', + default: false, + write: noop, + }, + { + name: 'missingBucketLabel', + default: i18n.translate('data.search.aggs.buckets.terms.missingBucketLabel', { + defaultMessage: 'Missing', + description: `Default label used in charts when documents are missing a field. + Visible when you create a chart with a terms aggregation and enable "Show missing values"`, + }), + type: 'string', + displayName: i18n.translate('data.search.aggs.otherBucket.labelForMissingValuesLabel', { + defaultMessage: 'Label for missing values', }), - value: 'asc', + shouldShow: agg => agg.getParam('missingBucket'), + write: noop, + }, + { + name: 'exclude', + displayName: i18n.translate('data.search.aggs.buckets.terms.excludeLabel', { + defaultMessage: 'Exclude', + }), + type: 'string', + advanced: true, + shouldShow: isStringType, + ...migrateIncludeExcludeFormat, + }, + { + name: 'include', + displayName: i18n.translate('data.search.aggs.buckets.terms.includeLabel', { + defaultMessage: 'Include', + }), + type: 'string', + advanced: true, + shouldShow: isStringType, + ...migrateIncludeExcludeFormat, }, ], - write: noop, // prevent default write, it's handled by orderAgg - }, - { - name: 'size', - default: 5, - }, - { - name: 'otherBucket', - default: false, - write: noop, - }, - { - name: 'otherBucketLabel', - type: 'string', - default: i18n.translate('data.search.aggs.buckets.terms.otherBucketLabel', { - defaultMessage: 'Other', - }), - displayName: i18n.translate('data.search.aggs.otherBucket.labelForOtherBucketLabel', { - defaultMessage: 'Label for other bucket', - }), - shouldShow: agg => agg.getParam('otherBucket'), - write: noop, - }, - { - name: 'missingBucket', - default: false, - write: noop, }, - { - name: 'missingBucketLabel', - default: i18n.translate('data.search.aggs.buckets.terms.missingBucketLabel', { - defaultMessage: 'Missing', - description: `Default label used in charts when documents are missing a field. - Visible when you create a chart with a terms aggregation and enable "Show missing values"`, - }), - type: 'string', - displayName: i18n.translate('data.search.aggs.otherBucket.labelForMissingValuesLabel', { - defaultMessage: 'Label for missing values', - }), - shouldShow: agg => agg.getParam('missingBucket'), - write: noop, - }, - { - name: 'exclude', - displayName: i18n.translate('data.search.aggs.buckets.terms.excludeLabel', { - defaultMessage: 'Exclude', - }), - type: 'string', - advanced: true, - shouldShow: isStringType, - ...migrateIncludeExcludeFormat, - }, - { - name: 'include', - displayName: i18n.translate('data.search.aggs.buckets.terms.includeLabel', { - defaultMessage: 'Include', - }), - type: 'string', - advanced: true, - shouldShow: isStringType, - ...migrateIncludeExcludeFormat, - }, - ], -}); + { getInternalStartServices } + ); diff --git a/src/plugins/data/public/search/aggs/index.test.ts b/src/plugins/data/public/search/aggs/index.test.ts index 8c0e47763c295..419c3fdab1caf 100644 --- a/src/plugins/data/public/search/aggs/index.test.ts +++ b/src/plugins/data/public/search/aggs/index.test.ts @@ -20,16 +20,22 @@ import { coreMock } from '../../../../../../src/core/public/mocks'; import { getAggTypes } from './index'; -import { isBucketAggType } from './buckets/_bucket_agg_type'; +import { isBucketAggType } from './buckets/bucket_agg_type'; import { isMetricAggType } from './metrics/metric_agg_type'; import { QueryStart } from '../../query'; +import { FieldFormatsStart } from '../../field_formats'; describe('AggTypesComponent', () => { - const core = coreMock.createSetup(); + const coreSetup = coreMock.createSetup(); + const coreStart = coreMock.createSetup(); + const aggTypes = getAggTypes({ - uiSettings: core.uiSettings, - notifications: core.notifications, + uiSettings: coreSetup.uiSettings, query: {} as QueryStart, + getInternalStartServices: () => ({ + notifications: coreStart.notifications, + fieldFormats: {} as FieldFormatsStart, + }), }); const { buckets, metrics } = aggTypes; diff --git a/src/plugins/data/public/search/aggs/metrics/avg.ts b/src/plugins/data/public/search/aggs/metrics/avg.ts index 008dede3e1985..d53ce8d3fc489 100644 --- a/src/plugins/data/public/search/aggs/metrics/avg.ts +++ b/src/plugins/data/public/search/aggs/metrics/avg.ts @@ -21,25 +21,37 @@ import { i18n } from '@kbn/i18n'; import { MetricAggType } from './metric_agg_type'; import { METRIC_TYPES } from './metric_agg_types'; import { KBN_FIELD_TYPES } from '../../../../common'; +import { GetInternalStartServicesFn } from '../../../types'; const averageTitle = i18n.translate('data.search.aggs.metrics.averageTitle', { defaultMessage: 'Average', }); -export const avgMetricAgg = new MetricAggType({ - name: METRIC_TYPES.AVG, - title: averageTitle, - makeLabel: aggConfig => { - return i18n.translate('data.search.aggs.metrics.averageLabel', { - defaultMessage: 'Average {field}', - values: { field: aggConfig.getFieldDisplayName() }, - }); - }, - params: [ +export interface AvgMetricAggDependencies { + getInternalStartServices: GetInternalStartServicesFn; +} + +export const getAvgMetricAgg = ({ getInternalStartServices }: AvgMetricAggDependencies) => { + return new MetricAggType( { - name: 'field', - type: 'field', - filterFieldTypes: KBN_FIELD_TYPES.NUMBER, + name: METRIC_TYPES.AVG, + title: averageTitle, + makeLabel: aggConfig => { + return i18n.translate('data.search.aggs.metrics.averageLabel', { + defaultMessage: 'Average {field}', + values: { field: aggConfig.getFieldDisplayName() }, + }); + }, + params: [ + { + name: 'field', + type: 'field', + filterFieldTypes: KBN_FIELD_TYPES.NUMBER, + }, + ], }, - ], -}); + { + getInternalStartServices, + } + ); +}; diff --git a/src/plugins/data/public/search/aggs/metrics/bucket_avg.ts b/src/plugins/data/public/search/aggs/metrics/bucket_avg.ts index 11bb559274729..2c32ebc671539 100644 --- a/src/plugins/data/public/search/aggs/metrics/bucket_avg.ts +++ b/src/plugins/data/public/search/aggs/metrics/bucket_avg.ts @@ -23,6 +23,11 @@ import { MetricAggType } from './metric_agg_type'; import { makeNestedLabel } from './lib/make_nested_label'; import { siblingPipelineAggHelper } from './lib/sibling_pipeline_agg_helper'; import { METRIC_TYPES } from './metric_agg_types'; +import { GetInternalStartServicesFn } from '../../../types'; + +export interface BucketAvgMetricAggDependencies { + getInternalStartServices: GetInternalStartServicesFn; +} const overallAverageLabel = i18n.translate('data.search.aggs.metrics.overallAverageLabel', { defaultMessage: 'overall average', @@ -32,25 +37,34 @@ const averageBucketTitle = i18n.translate('data.search.aggs.metrics.averageBucke defaultMessage: 'Average Bucket', }); -export const bucketAvgMetricAgg = new MetricAggType({ - name: METRIC_TYPES.AVG_BUCKET, - title: averageBucketTitle, - makeLabel: agg => makeNestedLabel(agg, overallAverageLabel), - subtype: siblingPipelineAggHelper.subtype, - params: [...siblingPipelineAggHelper.params()], - getFormat: siblingPipelineAggHelper.getFormat, - getValue(agg, bucket) { - const customMetric = agg.getParam('customMetric'); - const customBucket = agg.getParam('customBucket'); - const scaleMetrics = customMetric.type && customMetric.type.isScalable(); +export const getBucketAvgMetricAgg = ({ + getInternalStartServices, +}: BucketAvgMetricAggDependencies) => { + return new MetricAggType( + { + name: METRIC_TYPES.AVG_BUCKET, + title: averageBucketTitle, + makeLabel: agg => makeNestedLabel(agg, overallAverageLabel), + subtype: siblingPipelineAggHelper.subtype, + params: [...siblingPipelineAggHelper.params()], + getFormat: siblingPipelineAggHelper.getFormat, + getValue(agg, bucket) { + const customMetric = agg.getParam('customMetric'); + const customBucket = agg.getParam('customBucket'); + const scaleMetrics = customMetric.type && customMetric.type.isScalable(); - let value = bucket[agg.id] && bucket[agg.id].value; + let value = bucket[agg.id] && bucket[agg.id].value; - if (scaleMetrics && customBucket.type.name === 'date_histogram') { - const aggInfo = customBucket.write(); + if (scaleMetrics && customBucket.type.name === 'date_histogram') { + const aggInfo = customBucket.write(); - value *= get(aggInfo, 'bucketInterval.scale', 1); + value *= get(aggInfo, 'bucketInterval.scale', 1); + } + return value; + }, + }, + { + getInternalStartServices, } - return value; - }, -}); + ); +}; diff --git a/src/plugins/data/public/search/aggs/metrics/bucket_max.ts b/src/plugins/data/public/search/aggs/metrics/bucket_max.ts index 0668a9bcf57a8..1e57a2dd8e38e 100644 --- a/src/plugins/data/public/search/aggs/metrics/bucket_max.ts +++ b/src/plugins/data/public/search/aggs/metrics/bucket_max.ts @@ -22,6 +22,11 @@ import { MetricAggType } from './metric_agg_type'; import { makeNestedLabel } from './lib/make_nested_label'; import { siblingPipelineAggHelper } from './lib/sibling_pipeline_agg_helper'; import { METRIC_TYPES } from './metric_agg_types'; +import { GetInternalStartServicesFn } from '../../../types'; + +export interface BucketMaxMetricAggDependencies { + getInternalStartServices: GetInternalStartServicesFn; +} const overallMaxLabel = i18n.translate('data.search.aggs.metrics.overallMaxLabel', { defaultMessage: 'overall max', @@ -31,11 +36,20 @@ const maxBucketTitle = i18n.translate('data.search.aggs.metrics.maxBucketTitle', defaultMessage: 'Max Bucket', }); -export const bucketMaxMetricAgg = new MetricAggType({ - name: METRIC_TYPES.MAX_BUCKET, - title: maxBucketTitle, - makeLabel: agg => makeNestedLabel(agg, overallMaxLabel), - subtype: siblingPipelineAggHelper.subtype, - params: [...siblingPipelineAggHelper.params()], - getFormat: siblingPipelineAggHelper.getFormat, -}); +export const getBucketMaxMetricAgg = ({ + getInternalStartServices, +}: BucketMaxMetricAggDependencies) => { + return new MetricAggType( + { + name: METRIC_TYPES.MAX_BUCKET, + title: maxBucketTitle, + makeLabel: agg => makeNestedLabel(agg, overallMaxLabel), + subtype: siblingPipelineAggHelper.subtype, + params: [...siblingPipelineAggHelper.params()], + getFormat: siblingPipelineAggHelper.getFormat, + }, + { + getInternalStartServices, + } + ); +}; diff --git a/src/plugins/data/public/search/aggs/metrics/bucket_min.ts b/src/plugins/data/public/search/aggs/metrics/bucket_min.ts index 8f728cb5e7e42..0484af23a7141 100644 --- a/src/plugins/data/public/search/aggs/metrics/bucket_min.ts +++ b/src/plugins/data/public/search/aggs/metrics/bucket_min.ts @@ -22,6 +22,11 @@ import { MetricAggType } from './metric_agg_type'; import { makeNestedLabel } from './lib/make_nested_label'; import { siblingPipelineAggHelper } from './lib/sibling_pipeline_agg_helper'; import { METRIC_TYPES } from './metric_agg_types'; +import { GetInternalStartServicesFn } from '../../../types'; + +export interface BucketMinMetricAggDependencies { + getInternalStartServices: GetInternalStartServicesFn; +} const overallMinLabel = i18n.translate('data.search.aggs.metrics.overallMinLabel', { defaultMessage: 'overall min', @@ -31,11 +36,20 @@ const minBucketTitle = i18n.translate('data.search.aggs.metrics.minBucketTitle', defaultMessage: 'Min Bucket', }); -export const bucketMinMetricAgg = new MetricAggType({ - name: METRIC_TYPES.MIN_BUCKET, - title: minBucketTitle, - makeLabel: agg => makeNestedLabel(agg, overallMinLabel), - subtype: siblingPipelineAggHelper.subtype, - params: [...siblingPipelineAggHelper.params()], - getFormat: siblingPipelineAggHelper.getFormat, -}); +export const getBucketMinMetricAgg = ({ + getInternalStartServices, +}: BucketMinMetricAggDependencies) => { + return new MetricAggType( + { + name: METRIC_TYPES.MIN_BUCKET, + title: minBucketTitle, + makeLabel: agg => makeNestedLabel(agg, overallMinLabel), + subtype: siblingPipelineAggHelper.subtype, + params: [...siblingPipelineAggHelper.params()], + getFormat: siblingPipelineAggHelper.getFormat, + }, + { + getInternalStartServices, + } + ); +}; diff --git a/src/plugins/data/public/search/aggs/metrics/bucket_sum.ts b/src/plugins/data/public/search/aggs/metrics/bucket_sum.ts index 1f9392c5bec35..0a4d29a18a980 100644 --- a/src/plugins/data/public/search/aggs/metrics/bucket_sum.ts +++ b/src/plugins/data/public/search/aggs/metrics/bucket_sum.ts @@ -22,6 +22,11 @@ import { MetricAggType } from './metric_agg_type'; import { makeNestedLabel } from './lib/make_nested_label'; import { siblingPipelineAggHelper } from './lib/sibling_pipeline_agg_helper'; import { METRIC_TYPES } from './metric_agg_types'; +import { GetInternalStartServicesFn } from '../../../types'; + +export interface BucketSumMetricAggDependencies { + getInternalStartServices: GetInternalStartServicesFn; +} const overallSumLabel = i18n.translate('data.search.aggs.metrics.overallSumLabel', { defaultMessage: 'overall sum', @@ -31,11 +36,20 @@ const sumBucketTitle = i18n.translate('data.search.aggs.metrics.sumBucketTitle', defaultMessage: 'Sum Bucket', }); -export const bucketSumMetricAgg = new MetricAggType({ - name: METRIC_TYPES.SUM_BUCKET, - title: sumBucketTitle, - makeLabel: agg => makeNestedLabel(agg, overallSumLabel), - subtype: siblingPipelineAggHelper.subtype, - params: [...siblingPipelineAggHelper.params()], - getFormat: siblingPipelineAggHelper.getFormat, -}); +export const getBucketSumMetricAgg = ({ + getInternalStartServices, +}: BucketSumMetricAggDependencies) => { + return new MetricAggType( + { + name: METRIC_TYPES.SUM_BUCKET, + title: sumBucketTitle, + makeLabel: agg => makeNestedLabel(agg, overallSumLabel), + subtype: siblingPipelineAggHelper.subtype, + params: [...siblingPipelineAggHelper.params()], + getFormat: siblingPipelineAggHelper.getFormat, + }, + { + getInternalStartServices, + } + ); +}; diff --git a/src/plugins/data/public/search/aggs/metrics/cardinality.ts b/src/plugins/data/public/search/aggs/metrics/cardinality.ts index 88cdf3175665e..10b6b5aff1abd 100644 --- a/src/plugins/data/public/search/aggs/metrics/cardinality.ts +++ b/src/plugins/data/public/search/aggs/metrics/cardinality.ts @@ -18,36 +18,48 @@ */ import { i18n } from '@kbn/i18n'; -import { MetricAggType } from './metric_agg_type'; +import { MetricAggType, IMetricAggConfig } from './metric_agg_type'; import { METRIC_TYPES } from './metric_agg_types'; import { KBN_FIELD_TYPES } from '../../../../common'; -import { getFieldFormats } from '../../../../public/services'; +import { GetInternalStartServicesFn } from '../../../types'; const uniqueCountTitle = i18n.translate('data.search.aggs.metrics.uniqueCountTitle', { defaultMessage: 'Unique Count', }); -export const cardinalityMetricAgg = new MetricAggType({ - name: METRIC_TYPES.CARDINALITY, - title: uniqueCountTitle, - makeLabel(aggConfig) { - return i18n.translate('data.search.aggs.metrics.uniqueCountLabel', { - defaultMessage: 'Unique count of {field}', - values: { field: aggConfig.getFieldDisplayName() }, - }); - }, - getFormat() { - const fieldFormatsService = getFieldFormats(); +export interface CardinalityMetricAggDependencies { + getInternalStartServices: GetInternalStartServicesFn; +} - return fieldFormatsService.getDefaultInstance(KBN_FIELD_TYPES.NUMBER); - }, - params: [ +export const getCardinalityMetricAgg = ({ + getInternalStartServices, +}: CardinalityMetricAggDependencies) => + new MetricAggType( { - name: 'field', - type: 'field', - filterFieldTypes: Object.values(KBN_FIELD_TYPES).filter( - type => type !== KBN_FIELD_TYPES.HISTOGRAM - ), + name: METRIC_TYPES.CARDINALITY, + title: uniqueCountTitle, + makeLabel(aggConfig: IMetricAggConfig) { + return i18n.translate('data.search.aggs.metrics.uniqueCountLabel', { + defaultMessage: 'Unique count of {field}', + values: { field: aggConfig.getFieldDisplayName() }, + }); + }, + getFormat() { + const { fieldFormats } = getInternalStartServices(); + + return fieldFormats.getDefaultInstance(KBN_FIELD_TYPES.NUMBER); + }, + params: [ + { + name: 'field', + type: 'field', + filterFieldTypes: Object.values(KBN_FIELD_TYPES).filter( + type => type !== KBN_FIELD_TYPES.HISTOGRAM + ), + }, + ], }, - ], -}); + { + getInternalStartServices, + } + ); diff --git a/src/plugins/data/public/search/aggs/metrics/count.ts b/src/plugins/data/public/search/aggs/metrics/count.ts index 3ec1e18d66ab9..bd0b83798c7db 100644 --- a/src/plugins/data/public/search/aggs/metrics/count.ts +++ b/src/plugins/data/public/search/aggs/metrics/count.ts @@ -21,28 +21,38 @@ import { i18n } from '@kbn/i18n'; import { MetricAggType } from './metric_agg_type'; import { METRIC_TYPES } from './metric_agg_types'; import { KBN_FIELD_TYPES } from '../../../../common'; -import { getFieldFormats } from '../../../../public/services'; +import { GetInternalStartServicesFn } from '../../../types'; -export const countMetricAgg = new MetricAggType({ - name: METRIC_TYPES.COUNT, - title: i18n.translate('data.search.aggs.metrics.countTitle', { - defaultMessage: 'Count', - }), - hasNoDsl: true, - makeLabel() { - return i18n.translate('data.search.aggs.metrics.countLabel', { - defaultMessage: 'Count', - }); - }, - getFormat() { - const fieldFormatsService = getFieldFormats(); +export interface CountMetricAggDependencies { + getInternalStartServices: GetInternalStartServicesFn; +} - return fieldFormatsService.getDefaultInstance(KBN_FIELD_TYPES.NUMBER); - }, - getValue(agg, bucket) { - return bucket.doc_count; - }, - isScalable() { - return true; - }, -}); +export const getCountMetricAgg = ({ getInternalStartServices }: CountMetricAggDependencies) => + new MetricAggType( + { + name: METRIC_TYPES.COUNT, + title: i18n.translate('data.search.aggs.metrics.countTitle', { + defaultMessage: 'Count', + }), + hasNoDsl: true, + makeLabel() { + return i18n.translate('data.search.aggs.metrics.countLabel', { + defaultMessage: 'Count', + }); + }, + getFormat() { + const { fieldFormats } = getInternalStartServices(); + + return fieldFormats.getDefaultInstance(KBN_FIELD_TYPES.NUMBER); + }, + getValue(agg, bucket) { + return bucket.doc_count; + }, + isScalable() { + return true; + }, + }, + { + getInternalStartServices, + } + ); diff --git a/src/plugins/data/public/search/aggs/metrics/cumulative_sum.ts b/src/plugins/data/public/search/aggs/metrics/cumulative_sum.ts index a5d02459900bb..8ca922e144a1f 100644 --- a/src/plugins/data/public/search/aggs/metrics/cumulative_sum.ts +++ b/src/plugins/data/public/search/aggs/metrics/cumulative_sum.ts @@ -22,6 +22,11 @@ import { MetricAggType } from './metric_agg_type'; import { parentPipelineAggHelper } from './lib/parent_pipeline_agg_helper'; import { makeNestedLabel } from './lib/make_nested_label'; import { METRIC_TYPES } from './metric_agg_types'; +import { GetInternalStartServicesFn } from '../../../types'; + +export interface CumulativeSumMetricAggDependencies { + getInternalStartServices: GetInternalStartServicesFn; +} const cumulativeSumLabel = i18n.translate('data.search.aggs.metrics.cumulativeSumLabel', { defaultMessage: 'cumulative sum', @@ -31,11 +36,20 @@ const cumulativeSumTitle = i18n.translate('data.search.aggs.metrics.cumulativeSu defaultMessage: 'Cumulative Sum', }); -export const cumulativeSumMetricAgg = new MetricAggType({ - name: METRIC_TYPES.CUMULATIVE_SUM, - title: cumulativeSumTitle, - subtype: parentPipelineAggHelper.subtype, - makeLabel: agg => makeNestedLabel(agg, cumulativeSumLabel), - params: [...parentPipelineAggHelper.params()], - getFormat: parentPipelineAggHelper.getFormat, -}); +export const getCumulativeSumMetricAgg = ({ + getInternalStartServices, +}: CumulativeSumMetricAggDependencies) => { + return new MetricAggType( + { + name: METRIC_TYPES.CUMULATIVE_SUM, + title: cumulativeSumTitle, + subtype: parentPipelineAggHelper.subtype, + makeLabel: agg => makeNestedLabel(agg, cumulativeSumLabel), + params: [...parentPipelineAggHelper.params()], + getFormat: parentPipelineAggHelper.getFormat, + }, + { + getInternalStartServices, + } + ); +}; diff --git a/src/plugins/data/public/search/aggs/metrics/derivative.ts b/src/plugins/data/public/search/aggs/metrics/derivative.ts index 1169a527b0668..5752a72c846aa 100644 --- a/src/plugins/data/public/search/aggs/metrics/derivative.ts +++ b/src/plugins/data/public/search/aggs/metrics/derivative.ts @@ -22,6 +22,11 @@ import { MetricAggType } from './metric_agg_type'; import { parentPipelineAggHelper } from './lib/parent_pipeline_agg_helper'; import { makeNestedLabel } from './lib/make_nested_label'; import { METRIC_TYPES } from './metric_agg_types'; +import { GetInternalStartServicesFn } from '../../../types'; + +export interface DerivativeMetricAggDependencies { + getInternalStartServices: GetInternalStartServicesFn; +} const derivativeLabel = i18n.translate('data.search.aggs.metrics.derivativeLabel', { defaultMessage: 'derivative', @@ -31,13 +36,22 @@ const derivativeTitle = i18n.translate('data.search.aggs.metrics.derivativeTitle defaultMessage: 'Derivative', }); -export const derivativeMetricAgg = new MetricAggType({ - name: METRIC_TYPES.DERIVATIVE, - title: derivativeTitle, - subtype: parentPipelineAggHelper.subtype, - makeLabel(agg) { - return makeNestedLabel(agg, derivativeLabel); - }, - params: [...parentPipelineAggHelper.params()], - getFormat: parentPipelineAggHelper.getFormat, -}); +export const getDerivativeMetricAgg = ({ + getInternalStartServices, +}: DerivativeMetricAggDependencies) => { + return new MetricAggType( + { + name: METRIC_TYPES.DERIVATIVE, + title: derivativeTitle, + subtype: parentPipelineAggHelper.subtype, + makeLabel(agg) { + return makeNestedLabel(agg, derivativeLabel); + }, + params: [...parentPipelineAggHelper.params()], + getFormat: parentPipelineAggHelper.getFormat, + }, + { + getInternalStartServices, + } + ); +}; diff --git a/src/plugins/data/public/search/aggs/metrics/geo_bounds.ts b/src/plugins/data/public/search/aggs/metrics/geo_bounds.ts index 8a9f66f4b22a8..00927ebba56bf 100644 --- a/src/plugins/data/public/search/aggs/metrics/geo_bounds.ts +++ b/src/plugins/data/public/search/aggs/metrics/geo_bounds.ts @@ -21,6 +21,11 @@ import { i18n } from '@kbn/i18n'; import { MetricAggType } from './metric_agg_type'; import { METRIC_TYPES } from './metric_agg_types'; import { KBN_FIELD_TYPES } from '../../../../common'; +import { GetInternalStartServicesFn } from '../../../types'; + +export interface GeoBoundsMetricAggDependencies { + getInternalStartServices: GetInternalStartServicesFn; +} const geoBoundsTitle = i18n.translate('data.search.aggs.metrics.geoBoundsTitle', { defaultMessage: 'Geo Bounds', @@ -30,15 +35,24 @@ const geoBoundsLabel = i18n.translate('data.search.aggs.metrics.geoBoundsLabel', defaultMessage: 'Geo Bounds', }); -export const geoBoundsMetricAgg = new MetricAggType({ - name: METRIC_TYPES.GEO_BOUNDS, - title: geoBoundsTitle, - makeLabel: () => geoBoundsLabel, - params: [ +export const getGeoBoundsMetricAgg = ({ + getInternalStartServices, +}: GeoBoundsMetricAggDependencies) => { + return new MetricAggType( { - name: 'field', - type: 'field', - filterFieldTypes: KBN_FIELD_TYPES.GEO_POINT, + name: METRIC_TYPES.GEO_BOUNDS, + title: geoBoundsTitle, + makeLabel: () => geoBoundsLabel, + params: [ + { + name: 'field', + type: 'field', + filterFieldTypes: KBN_FIELD_TYPES.GEO_POINT, + }, + ], }, - ], -}); + { + getInternalStartServices, + } + ); +}; diff --git a/src/plugins/data/public/search/aggs/metrics/geo_centroid.ts b/src/plugins/data/public/search/aggs/metrics/geo_centroid.ts index a4e4413843bdd..a4b084f794a5d 100644 --- a/src/plugins/data/public/search/aggs/metrics/geo_centroid.ts +++ b/src/plugins/data/public/search/aggs/metrics/geo_centroid.ts @@ -21,6 +21,11 @@ import { i18n } from '@kbn/i18n'; import { MetricAggType } from './metric_agg_type'; import { METRIC_TYPES } from './metric_agg_types'; import { KBN_FIELD_TYPES } from '../../../../common'; +import { GetInternalStartServicesFn } from '../../../types'; + +export interface GeoCentroidMetricAggDependencies { + getInternalStartServices: GetInternalStartServicesFn; +} const geoCentroidTitle = i18n.translate('data.search.aggs.metrics.geoCentroidTitle', { defaultMessage: 'Geo Centroid', @@ -30,18 +35,27 @@ const geoCentroidLabel = i18n.translate('data.search.aggs.metrics.geoCentroidLab defaultMessage: 'Geo Centroid', }); -export const geoCentroidMetricAgg = new MetricAggType({ - name: METRIC_TYPES.GEO_CENTROID, - title: geoCentroidTitle, - makeLabel: () => geoCentroidLabel, - params: [ +export const getGeoCentroidMetricAgg = ({ + getInternalStartServices, +}: GeoCentroidMetricAggDependencies) => { + return new MetricAggType( { - name: 'field', - type: 'field', - filterFieldTypes: KBN_FIELD_TYPES.GEO_POINT, + name: METRIC_TYPES.GEO_CENTROID, + title: geoCentroidTitle, + makeLabel: () => geoCentroidLabel, + params: [ + { + name: 'field', + type: 'field', + filterFieldTypes: KBN_FIELD_TYPES.GEO_POINT, + }, + ], + getValue(agg, bucket) { + return bucket[agg.id] && bucket[agg.id].location; + }, }, - ], - getValue(agg, bucket) { - return bucket[agg.id] && bucket[agg.id].location; - }, -}); + { + getInternalStartServices, + } + ); +}; diff --git a/src/plugins/data/public/search/aggs/metrics/max.ts b/src/plugins/data/public/search/aggs/metrics/max.ts index 0cfb7be699a95..88e8b485cb73f 100644 --- a/src/plugins/data/public/search/aggs/metrics/max.ts +++ b/src/plugins/data/public/search/aggs/metrics/max.ts @@ -21,25 +21,37 @@ import { i18n } from '@kbn/i18n'; import { MetricAggType } from './metric_agg_type'; import { METRIC_TYPES } from './metric_agg_types'; import { KBN_FIELD_TYPES } from '../../../../common'; +import { GetInternalStartServicesFn } from '../../../types'; const maxTitle = i18n.translate('data.search.aggs.metrics.maxTitle', { defaultMessage: 'Max', }); -export const maxMetricAgg = new MetricAggType({ - name: METRIC_TYPES.MAX, - title: maxTitle, - makeLabel(aggConfig) { - return i18n.translate('data.search.aggs.metrics.maxLabel', { - defaultMessage: 'Max {field}', - values: { field: aggConfig.getFieldDisplayName() }, - }); - }, - params: [ +export interface MaxMetricAggDependencies { + getInternalStartServices: GetInternalStartServicesFn; +} + +export const getMaxMetricAgg = ({ getInternalStartServices }: MaxMetricAggDependencies) => { + return new MetricAggType( { - name: 'field', - type: 'field', - filterFieldTypes: [KBN_FIELD_TYPES.NUMBER, KBN_FIELD_TYPES.DATE], + name: METRIC_TYPES.MAX, + title: maxTitle, + makeLabel(aggConfig) { + return i18n.translate('data.search.aggs.metrics.maxLabel', { + defaultMessage: 'Max {field}', + values: { field: aggConfig.getFieldDisplayName() }, + }); + }, + params: [ + { + name: 'field', + type: 'field', + filterFieldTypes: [KBN_FIELD_TYPES.NUMBER, KBN_FIELD_TYPES.DATE], + }, + ], }, - ], -}); + { + getInternalStartServices, + } + ); +}; diff --git a/src/plugins/data/public/search/aggs/metrics/median.test.ts b/src/plugins/data/public/search/aggs/metrics/median.test.ts index ad55837ec9a30..f80c46026f50a 100644 --- a/src/plugins/data/public/search/aggs/metrics/median.test.ts +++ b/src/plugins/data/public/search/aggs/metrics/median.test.ts @@ -17,16 +17,24 @@ * under the License. */ -import { medianMetricAgg } from './median'; +import { getMedianMetricAgg, MedianMetricAggDependencies } from './median'; import { AggConfigs, IAggConfigs } from '../agg_configs'; import { mockAggTypesRegistry } from '../test_helpers'; import { METRIC_TYPES } from './metric_agg_types'; +import { fieldFormatsServiceMock } from '../../../field_formats/mocks'; +import { notificationServiceMock } from '../../../../../../../src/core/public/mocks'; describe('AggTypeMetricMedianProvider class', () => { let aggConfigs: IAggConfigs; + const aggTypesDependencies: MedianMetricAggDependencies = { + getInternalStartServices: () => ({ + fieldFormats: fieldFormatsServiceMock.createStartContract(), + notifications: notificationServiceMock.createStartContract(), + }), + }; beforeEach(() => { - const typesRegistry = mockAggTypesRegistry([medianMetricAgg]); + const typesRegistry = mockAggTypesRegistry([getMedianMetricAgg(aggTypesDependencies)]); const field = { name: 'bytes', }; diff --git a/src/plugins/data/public/search/aggs/metrics/median.ts b/src/plugins/data/public/search/aggs/metrics/median.ts index faa0694cd5312..a398f017602b0 100644 --- a/src/plugins/data/public/search/aggs/metrics/median.ts +++ b/src/plugins/data/public/search/aggs/metrics/median.ts @@ -21,33 +21,49 @@ import { i18n } from '@kbn/i18n'; import { MetricAggType } from './metric_agg_type'; import { METRIC_TYPES } from './metric_agg_types'; import { KBN_FIELD_TYPES } from '../../../../common'; +import { GetInternalStartServicesFn } from '../../../types'; const medianTitle = i18n.translate('data.search.aggs.metrics.medianTitle', { defaultMessage: 'Median', }); -export const medianMetricAgg = new MetricAggType({ - name: METRIC_TYPES.MEDIAN, - dslName: 'percentiles', - title: medianTitle, - makeLabel(aggConfig) { - return i18n.translate('data.search.aggs.metrics.medianLabel', { - defaultMessage: 'Median {field}', - values: { field: aggConfig.getFieldDisplayName() }, - }); - }, - params: [ +export interface MedianMetricAggDependencies { + getInternalStartServices: GetInternalStartServicesFn; +} + +export const getMedianMetricAgg = ({ getInternalStartServices }: MedianMetricAggDependencies) => { + return new MetricAggType( { - name: 'field', - type: 'field', - filterFieldTypes: [KBN_FIELD_TYPES.NUMBER, KBN_FIELD_TYPES.DATE, KBN_FIELD_TYPES.HISTOGRAM], - write(agg, output) { - output.params.field = agg.getParam('field').name; - output.params.percents = [50]; + name: METRIC_TYPES.MEDIAN, + dslName: 'percentiles', + title: medianTitle, + makeLabel(aggConfig) { + return i18n.translate('data.search.aggs.metrics.medianLabel', { + defaultMessage: 'Median {field}', + values: { field: aggConfig.getFieldDisplayName() }, + }); + }, + params: [ + { + name: 'field', + type: 'field', + filterFieldTypes: [ + KBN_FIELD_TYPES.NUMBER, + KBN_FIELD_TYPES.DATE, + KBN_FIELD_TYPES.HISTOGRAM, + ], + write(agg, output) { + output.params.field = agg.getParam('field').name; + output.params.percents = [50]; + }, + }, + ], + getValue(agg, bucket) { + return bucket[agg.id].values['50.0']; }, }, - ], - getValue(agg, bucket) { - return bucket[agg.id].values['50.0']; - }, -}); + { + getInternalStartServices, + } + ); +}; diff --git a/src/plugins/data/public/search/aggs/metrics/metric_agg_type.ts b/src/plugins/data/public/search/aggs/metrics/metric_agg_type.ts index 05c4cb3de4bdf..bb16cba1bee62 100644 --- a/src/plugins/data/public/search/aggs/metrics/metric_agg_type.ts +++ b/src/plugins/data/public/search/aggs/metrics/metric_agg_type.ts @@ -23,8 +23,8 @@ import { AggParamType } from '../param_types/agg'; import { AggConfig } from '../agg_config'; import { METRIC_TYPES } from './metric_agg_types'; import { KBN_FIELD_TYPES } from '../../../../common'; -import { getFieldFormats } from '../../../../public/services'; import { FieldTypes } from '../param_types'; +import { GetInternalStartServicesFn } from '../../../types'; export interface IMetricAggConfig extends AggConfig { type: InstanceType; @@ -44,6 +44,10 @@ interface MetricAggTypeConfig subtype?: string; } +interface MetricAggTypeDependencies { + getInternalStartServices: GetInternalStartServicesFn; +} + // TODO need to make a more explicit interface for this export type IMetricAggType = MetricAggType; @@ -57,8 +61,11 @@ export class MetricAggType {}; - constructor(config: MetricAggTypeConfig) { - super(config); + constructor( + config: MetricAggTypeConfig, + dependencies: MetricAggTypeDependencies + ) { + super(config, dependencies); this.getValue = config.getValue || @@ -78,11 +85,9 @@ export class MetricAggType { - const fieldFormatsService = getFieldFormats(); + const { fieldFormats } = dependencies.getInternalStartServices(); const field = agg.getField(); - return field - ? field.format - : fieldFormatsService.getDefaultInstance(KBN_FIELD_TYPES.NUMBER); + return field ? field.format : fieldFormats.getDefaultInstance(KBN_FIELD_TYPES.NUMBER); }); this.subtype = diff --git a/src/plugins/data/public/search/aggs/metrics/min.ts b/src/plugins/data/public/search/aggs/metrics/min.ts index 0a9abf1edcd04..aae16f357186c 100644 --- a/src/plugins/data/public/search/aggs/metrics/min.ts +++ b/src/plugins/data/public/search/aggs/metrics/min.ts @@ -21,25 +21,37 @@ import { i18n } from '@kbn/i18n'; import { MetricAggType } from './metric_agg_type'; import { METRIC_TYPES } from './metric_agg_types'; import { KBN_FIELD_TYPES } from '../../../../common'; +import { GetInternalStartServicesFn } from '../../../types'; const minTitle = i18n.translate('data.search.aggs.metrics.minTitle', { defaultMessage: 'Min', }); -export const minMetricAgg = new MetricAggType({ - name: METRIC_TYPES.MIN, - title: minTitle, - makeLabel(aggConfig) { - return i18n.translate('data.search.aggs.metrics.minLabel', { - defaultMessage: 'Min {field}', - values: { field: aggConfig.getFieldDisplayName() }, - }); - }, - params: [ +export interface MinMetricAggDependencies { + getInternalStartServices: GetInternalStartServicesFn; +} + +export const getMinMetricAgg = ({ getInternalStartServices }: MinMetricAggDependencies) => { + return new MetricAggType( { - name: 'field', - type: 'field', - filterFieldTypes: [KBN_FIELD_TYPES.NUMBER, KBN_FIELD_TYPES.DATE], + name: METRIC_TYPES.MIN, + title: minTitle, + makeLabel(aggConfig) { + return i18n.translate('data.search.aggs.metrics.minLabel', { + defaultMessage: 'Min {field}', + values: { field: aggConfig.getFieldDisplayName() }, + }); + }, + params: [ + { + name: 'field', + type: 'field', + filterFieldTypes: [KBN_FIELD_TYPES.NUMBER, KBN_FIELD_TYPES.DATE], + }, + ], }, - ], -}); + { + getInternalStartServices, + } + ); +}; diff --git a/src/plugins/data/public/search/aggs/metrics/moving_avg.ts b/src/plugins/data/public/search/aggs/metrics/moving_avg.ts index cb733507858bc..94b9b1d8cd487 100644 --- a/src/plugins/data/public/search/aggs/metrics/moving_avg.ts +++ b/src/plugins/data/public/search/aggs/metrics/moving_avg.ts @@ -22,6 +22,11 @@ import { MetricAggType } from './metric_agg_type'; import { parentPipelineAggHelper } from './lib/parent_pipeline_agg_helper'; import { makeNestedLabel } from './lib/make_nested_label'; import { METRIC_TYPES } from './metric_agg_types'; +import { GetInternalStartServicesFn } from '../../../types'; + +export interface MovingAvgMetricAggDependencies { + getInternalStartServices: GetInternalStartServicesFn; +} const movingAvgTitle = i18n.translate('data.search.aggs.metrics.movingAvgTitle', { defaultMessage: 'Moving Avg', @@ -31,34 +36,43 @@ const movingAvgLabel = i18n.translate('data.search.aggs.metrics.movingAvgLabel', defaultMessage: 'moving avg', }); -export const movingAvgMetricAgg = new MetricAggType({ - name: METRIC_TYPES.MOVING_FN, - dslName: 'moving_fn', - title: movingAvgTitle, - subtype: parentPipelineAggHelper.subtype, - makeLabel: agg => makeNestedLabel(agg, movingAvgLabel), - params: [ - ...parentPipelineAggHelper.params(), +export const getMovingAvgMetricAgg = ({ + getInternalStartServices, +}: MovingAvgMetricAggDependencies) => { + return new MetricAggType( { - name: 'window', - default: 5, + name: METRIC_TYPES.MOVING_FN, + dslName: 'moving_fn', + title: movingAvgTitle, + subtype: parentPipelineAggHelper.subtype, + makeLabel: agg => makeNestedLabel(agg, movingAvgLabel), + params: [ + ...parentPipelineAggHelper.params(), + { + name: 'window', + default: 5, + }, + { + name: 'script', + default: 'MovingFunctions.unweightedAvg(values)', + }, + ], + getValue(agg, bucket) { + /** + * The previous implementation using `moving_avg` did not + * return any bucket in case there are no documents or empty window. + * The `moving_fn` aggregation returns buckets with the value null if the + * window is empty or doesn't return any value if the sibiling metric + * is null. Since our generic MetricAggType.getValue implementation + * would return the value 0 for null buckets, we need a specific + * implementation here, that preserves the null value. + */ + return bucket[agg.id] ? bucket[agg.id].value : null; + }, + getFormat: parentPipelineAggHelper.getFormat, }, { - name: 'script', - default: 'MovingFunctions.unweightedAvg(values)', - }, - ], - getValue(agg, bucket) { - /** - * The previous implementation using `moving_avg` did not - * return any bucket in case there are no documents or empty window. - * The `moving_fn` aggregation returns buckets with the value null if the - * window is empty or doesn't return any value if the sibiling metric - * is null. Since our generic MetricAggType.getValue implementation - * would return the value 0 for null buckets, we need a specific - * implementation here, that preserves the null value. - */ - return bucket[agg.id] ? bucket[agg.id].value : null; - }, - getFormat: parentPipelineAggHelper.getFormat, -}); + getInternalStartServices, + } + ); +}; diff --git a/src/plugins/data/public/search/aggs/metrics/parent_pipeline.test.ts b/src/plugins/data/public/search/aggs/metrics/parent_pipeline.test.ts index 02e63f653f94f..af983a50f6c23 100644 --- a/src/plugins/data/public/search/aggs/metrics/parent_pipeline.test.ts +++ b/src/plugins/data/public/search/aggs/metrics/parent_pipeline.test.ts @@ -17,26 +17,47 @@ * under the License. */ -import { derivativeMetricAgg } from './derivative'; -import { cumulativeSumMetricAgg } from './cumulative_sum'; -import { movingAvgMetricAgg } from './moving_avg'; -import { serialDiffMetricAgg } from './serial_diff'; +import { getDerivativeMetricAgg } from './derivative'; +import { getCumulativeSumMetricAgg } from './cumulative_sum'; +import { getMovingAvgMetricAgg } from './moving_avg'; +import { getSerialDiffMetricAgg } from './serial_diff'; import { AggConfigs } from '../agg_configs'; -import { mockDataServices, mockAggTypesRegistry } from '../test_helpers'; +import { mockAggTypesRegistry } from '../test_helpers'; import { IMetricAggConfig, MetricAggType } from './metric_agg_type'; +import { fieldFormatsServiceMock } from '../../../field_formats/mocks'; +import { GetInternalStartServicesFn } from '../../../types'; +import { notificationServiceMock } from '../../../../../../../src/core/public/mocks'; describe('parent pipeline aggs', function() { - beforeEach(() => { - mockDataServices(); + const getInternalStartServices: GetInternalStartServicesFn = () => ({ + fieldFormats: fieldFormatsServiceMock.createStartContract(), + notifications: notificationServiceMock.createStartContract(), }); const typesRegistry = mockAggTypesRegistry(); const metrics = [ - { name: 'derivative', title: 'Derivative', provider: derivativeMetricAgg }, - { name: 'cumulative_sum', title: 'Cumulative Sum', provider: cumulativeSumMetricAgg }, - { name: 'moving_avg', title: 'Moving Avg', provider: movingAvgMetricAgg, dslName: 'moving_fn' }, - { name: 'serial_diff', title: 'Serial Diff', provider: serialDiffMetricAgg }, + { + name: 'derivative', + title: 'Derivative', + provider: getDerivativeMetricAgg({ getInternalStartServices }), + }, + { + name: 'cumulative_sum', + title: 'Cumulative Sum', + provider: getCumulativeSumMetricAgg({ getInternalStartServices }), + }, + { + name: 'moving_avg', + title: 'Moving Avg', + provider: getMovingAvgMetricAgg({ getInternalStartServices }), + dslName: 'moving_fn', + }, + { + name: 'serial_diff', + title: 'Serial Diff', + provider: getSerialDiffMetricAgg({ getInternalStartServices }), + }, ]; metrics.forEach(metric => { diff --git a/src/plugins/data/public/search/aggs/metrics/percentile_ranks.test.ts b/src/plugins/data/public/search/aggs/metrics/percentile_ranks.test.ts index 628f1cd204ee5..2944fc8c11b23 100644 --- a/src/plugins/data/public/search/aggs/metrics/percentile_ranks.test.ts +++ b/src/plugins/data/public/search/aggs/metrics/percentile_ranks.test.ts @@ -17,18 +17,28 @@ * under the License. */ -import { IPercentileRanksAggConfig, percentileRanksMetricAgg } from './percentile_ranks'; +import { + IPercentileRanksAggConfig, + getPercentileRanksMetricAgg, + PercentileRanksMetricAggDependencies, +} from './percentile_ranks'; import { AggConfigs, IAggConfigs } from '../agg_configs'; -import { mockDataServices, mockAggTypesRegistry } from '../test_helpers'; +import { mockAggTypesRegistry } from '../test_helpers'; import { METRIC_TYPES } from './metric_agg_types'; +import { fieldFormatsServiceMock } from '../../../field_formats/mocks'; +import { notificationServiceMock } from '../../../../../../../src/core/public/mocks'; describe('AggTypesMetricsPercentileRanksProvider class', function() { let aggConfigs: IAggConfigs; + const aggTypesDependencies: PercentileRanksMetricAggDependencies = { + getInternalStartServices: () => ({ + fieldFormats: fieldFormatsServiceMock.createStartContract(), + notifications: notificationServiceMock.createStartContract(), + }), + }; beforeEach(() => { - mockDataServices(); - - const typesRegistry = mockAggTypesRegistry([percentileRanksMetricAgg]); + const typesRegistry = mockAggTypesRegistry([getPercentileRanksMetricAgg(aggTypesDependencies)]); const field = { name: 'bytes', }; @@ -65,7 +75,7 @@ describe('AggTypesMetricsPercentileRanksProvider class', function() { }); it('uses the custom label if it is set', function() { - const responseAggs: any = percentileRanksMetricAgg.getResponseAggs( + const responseAggs: any = getPercentileRanksMetricAgg(aggTypesDependencies).getResponseAggs( aggConfigs.aggs[0] as IPercentileRanksAggConfig ); diff --git a/src/plugins/data/public/search/aggs/metrics/percentile_ranks.ts b/src/plugins/data/public/search/aggs/metrics/percentile_ranks.ts index 7dc0f70ea7b80..0d79665ff9c4e 100644 --- a/src/plugins/data/public/search/aggs/metrics/percentile_ranks.ts +++ b/src/plugins/data/public/search/aggs/metrics/percentile_ranks.ts @@ -23,68 +23,86 @@ import { getResponseAggConfigClass, IResponseAggConfig } from './lib/get_respons import { getPercentileValue } from './percentiles_get_value'; import { METRIC_TYPES } from './metric_agg_types'; import { FIELD_FORMAT_IDS, KBN_FIELD_TYPES } from '../../../../common'; -import { getFieldFormats } from '../../../../public/services'; +import { GetInternalStartServicesFn } from '../../../types'; // required by the values editor export type IPercentileRanksAggConfig = IResponseAggConfig; -const valueProps = { - makeLabel(this: IPercentileRanksAggConfig) { - const fieldFormatsService = getFieldFormats(); - const field = this.getField(); - const format = - (field && field.format) || fieldFormatsService.getDefaultInstance(KBN_FIELD_TYPES.NUMBER); - const customLabel = this.getParam('customLabel'); - const label = customLabel || this.getFieldDisplayName(); +export interface PercentileRanksMetricAggDependencies { + getInternalStartServices: GetInternalStartServicesFn; +} - return i18n.translate('data.search.aggs.metrics.percentileRanks.valuePropsLabel', { - defaultMessage: 'Percentile rank {format} of "{label}"', - values: { format: format.convert(this.key, 'text'), label }, - }); - }, -}; +const getValueProps = (getInternalStartServices: GetInternalStartServicesFn) => { + return { + makeLabel(this: IPercentileRanksAggConfig) { + const { fieldFormats } = getInternalStartServices(); + const field = this.getField(); + const format = + (field && field.format) || fieldFormats.getDefaultInstance(KBN_FIELD_TYPES.NUMBER); + const customLabel = this.getParam('customLabel'); + const label = customLabel || this.getFieldDisplayName(); -export const percentileRanksMetricAgg = new MetricAggType({ - name: METRIC_TYPES.PERCENTILE_RANKS, - title: i18n.translate('data.search.aggs.metrics.percentileRanksTitle', { - defaultMessage: 'Percentile Ranks', - }), - makeLabel(agg) { - return i18n.translate('data.search.aggs.metrics.percentileRanksLabel', { - defaultMessage: 'Percentile ranks of {field}', - values: { field: agg.getFieldDisplayName() }, - }); - }, - params: [ - { - name: 'field', - type: 'field', - filterFieldTypes: [KBN_FIELD_TYPES.NUMBER, KBN_FIELD_TYPES.HISTOGRAM], - }, - { - name: 'values', - default: [], + return i18n.translate('data.search.aggs.metrics.percentileRanks.valuePropsLabel', { + defaultMessage: 'Percentile rank {format} of "{label}"', + values: { format: format.convert(this.key, 'text'), label }, + }); }, + }; +}; + +export const getPercentileRanksMetricAgg = ({ + getInternalStartServices, +}: PercentileRanksMetricAggDependencies) => { + return new MetricAggType( { - write(agg, output) { - output.params.keyed = false; + name: METRIC_TYPES.PERCENTILE_RANKS, + title: i18n.translate('data.search.aggs.metrics.percentileRanksTitle', { + defaultMessage: 'Percentile Ranks', + }), + makeLabel(agg) { + return i18n.translate('data.search.aggs.metrics.percentileRanksLabel', { + defaultMessage: 'Percentile ranks of {field}', + values: { field: agg.getFieldDisplayName() }, + }); }, - }, - ], - getResponseAggs(agg) { - const ValueAggConfig = getResponseAggConfigClass(agg, valueProps); - const values = agg.getParam('values'); + params: [ + { + name: 'field', + type: 'field', + filterFieldTypes: [KBN_FIELD_TYPES.NUMBER, KBN_FIELD_TYPES.HISTOGRAM], + }, + { + name: 'values', + default: [], + }, + { + write(agg, output) { + output.params.keyed = false; + }, + }, + ], + getResponseAggs(agg) { + const ValueAggConfig = getResponseAggConfigClass( + agg, + getValueProps(getInternalStartServices) + ); + const values = agg.getParam('values'); - return values.map((value: any) => new ValueAggConfig(value)); - }, - getFormat() { - const fieldFormatsService = getFieldFormats(); - return ( - fieldFormatsService.getInstance(FIELD_FORMAT_IDS.PERCENT) || - fieldFormatsService.getDefaultInstance(KBN_FIELD_TYPES.NUMBER) - ); - }, - getValue(agg, bucket) { - return getPercentileValue(agg, bucket) / 100; - }, -}); + return values.map((value: any) => new ValueAggConfig(value)); + }, + getFormat() { + const { fieldFormats } = getInternalStartServices(); + return ( + fieldFormats.getInstance(FIELD_FORMAT_IDS.PERCENT) || + fieldFormats.getDefaultInstance(KBN_FIELD_TYPES.NUMBER) + ); + }, + getValue(agg, bucket) { + return getPercentileValue(agg, bucket) / 100; + }, + }, + { + getInternalStartServices, + } + ); +}; diff --git a/src/plugins/data/public/search/aggs/metrics/percentiles.test.ts b/src/plugins/data/public/search/aggs/metrics/percentiles.test.ts index e077bc0f8c773..33bd42df74cc7 100644 --- a/src/plugins/data/public/search/aggs/metrics/percentiles.test.ts +++ b/src/plugins/data/public/search/aggs/metrics/percentiles.test.ts @@ -17,16 +17,28 @@ * under the License. */ -import { IPercentileAggConfig, percentilesMetricAgg } from './percentiles'; +import { + IPercentileAggConfig, + getPercentilesMetricAgg, + PercentilesMetricAggDependencies, +} from './percentiles'; import { AggConfigs, IAggConfigs } from '../agg_configs'; import { mockAggTypesRegistry } from '../test_helpers'; import { METRIC_TYPES } from './metric_agg_types'; +import { fieldFormatsServiceMock } from '../../../field_formats/mocks'; +import { notificationServiceMock } from '../../../../../../../src/core/public/mocks'; describe('AggTypesMetricsPercentilesProvider class', () => { let aggConfigs: IAggConfigs; + const aggTypesDependencies: PercentilesMetricAggDependencies = { + getInternalStartServices: () => ({ + fieldFormats: fieldFormatsServiceMock.createStartContract(), + notifications: notificationServiceMock.createStartContract(), + }), + }; beforeEach(() => { - const typesRegistry = mockAggTypesRegistry([percentilesMetricAgg]); + const typesRegistry = mockAggTypesRegistry([getPercentilesMetricAgg(aggTypesDependencies)]); const field = { name: 'bytes', }; @@ -63,7 +75,7 @@ describe('AggTypesMetricsPercentilesProvider class', () => { }); it('uses the custom label if it is set', () => { - const responseAggs: any = percentilesMetricAgg.getResponseAggs( + const responseAggs: any = getPercentilesMetricAgg(aggTypesDependencies).getResponseAggs( aggConfigs.aggs[0] as IPercentileAggConfig ); diff --git a/src/plugins/data/public/search/aggs/metrics/percentiles.ts b/src/plugins/data/public/search/aggs/metrics/percentiles.ts index a39d68248d608..040a52588dd94 100644 --- a/src/plugins/data/public/search/aggs/metrics/percentiles.ts +++ b/src/plugins/data/public/search/aggs/metrics/percentiles.ts @@ -24,9 +24,14 @@ import { KBN_FIELD_TYPES } from '../../../../common'; import { getResponseAggConfigClass, IResponseAggConfig } from './lib/get_response_agg_config_class'; import { getPercentileValue } from './percentiles_get_value'; import { ordinalSuffix } from './lib/ordinal_suffix'; +import { GetInternalStartServicesFn } from '../../../types'; export type IPercentileAggConfig = IResponseAggConfig; +export interface PercentilesMetricAggDependencies { + getInternalStartServices: GetInternalStartServicesFn; +} + const valueProps = { makeLabel(this: IPercentileAggConfig) { const customLabel = this.getParam('customLabel'); @@ -39,38 +44,51 @@ const valueProps = { }, }; -export const percentilesMetricAgg = new MetricAggType({ - name: METRIC_TYPES.PERCENTILES, - title: i18n.translate('data.search.aggs.metrics.percentilesTitle', { - defaultMessage: 'Percentiles', - }), - makeLabel(agg) { - return i18n.translate('data.search.aggs.metrics.percentilesLabel', { - defaultMessage: 'Percentiles of {field}', - values: { field: agg.getFieldDisplayName() }, - }); - }, - params: [ - { - name: 'field', - type: 'field', - filterFieldTypes: [KBN_FIELD_TYPES.NUMBER, KBN_FIELD_TYPES.DATE, KBN_FIELD_TYPES.HISTOGRAM], - }, - { - name: 'percents', - default: [1, 5, 25, 50, 75, 95, 99], - }, +export const getPercentilesMetricAgg = ({ + getInternalStartServices, +}: PercentilesMetricAggDependencies) => { + return new MetricAggType( { - write(agg, output) { - output.params.keyed = false; + name: METRIC_TYPES.PERCENTILES, + title: i18n.translate('data.search.aggs.metrics.percentilesTitle', { + defaultMessage: 'Percentiles', + }), + makeLabel(agg) { + return i18n.translate('data.search.aggs.metrics.percentilesLabel', { + defaultMessage: 'Percentiles of {field}', + values: { field: agg.getFieldDisplayName() }, + }); }, - }, - ], - getResponseAggs(agg) { - const ValueAggConfig = getResponseAggConfigClass(agg, valueProps); + params: [ + { + name: 'field', + type: 'field', + filterFieldTypes: [ + KBN_FIELD_TYPES.NUMBER, + KBN_FIELD_TYPES.DATE, + KBN_FIELD_TYPES.HISTOGRAM, + ], + }, + { + name: 'percents', + default: [1, 5, 25, 50, 75, 95, 99], + }, + { + write(agg, output) { + output.params.keyed = false; + }, + }, + ], + getResponseAggs(agg) { + const ValueAggConfig = getResponseAggConfigClass(agg, valueProps); - return agg.getParam('percents').map((percent: any) => new ValueAggConfig(percent)); - }, + return agg.getParam('percents').map((percent: any) => new ValueAggConfig(percent)); + }, - getValue: getPercentileValue, -}); + getValue: getPercentileValue, + }, + { + getInternalStartServices, + } + ); +}; diff --git a/src/plugins/data/public/search/aggs/metrics/serial_diff.ts b/src/plugins/data/public/search/aggs/metrics/serial_diff.ts index 5af6e1952d135..2b1498560f862 100644 --- a/src/plugins/data/public/search/aggs/metrics/serial_diff.ts +++ b/src/plugins/data/public/search/aggs/metrics/serial_diff.ts @@ -22,6 +22,11 @@ import { MetricAggType } from './metric_agg_type'; import { parentPipelineAggHelper } from './lib/parent_pipeline_agg_helper'; import { makeNestedLabel } from './lib/make_nested_label'; import { METRIC_TYPES } from './metric_agg_types'; +import { GetInternalStartServicesFn } from '../../../types'; + +export interface SerialDiffMetricAggDependencies { + getInternalStartServices: GetInternalStartServicesFn; +} const serialDiffTitle = i18n.translate('data.search.aggs.metrics.serialDiffTitle', { defaultMessage: 'Serial Diff', @@ -31,11 +36,20 @@ const serialDiffLabel = i18n.translate('data.search.aggs.metrics.serialDiffLabel defaultMessage: 'serial diff', }); -export const serialDiffMetricAgg = new MetricAggType({ - name: METRIC_TYPES.SERIAL_DIFF, - title: serialDiffTitle, - subtype: parentPipelineAggHelper.subtype, - makeLabel: agg => makeNestedLabel(agg, serialDiffLabel), - params: [...parentPipelineAggHelper.params()], - getFormat: parentPipelineAggHelper.getFormat, -}); +export const getSerialDiffMetricAgg = ({ + getInternalStartServices, +}: SerialDiffMetricAggDependencies) => { + return new MetricAggType( + { + name: METRIC_TYPES.SERIAL_DIFF, + title: serialDiffTitle, + subtype: parentPipelineAggHelper.subtype, + makeLabel: agg => makeNestedLabel(agg, serialDiffLabel), + params: [...parentPipelineAggHelper.params()], + getFormat: parentPipelineAggHelper.getFormat, + }, + { + getInternalStartServices, + } + ); +}; diff --git a/src/plugins/data/public/search/aggs/metrics/sibling_pipeline.test.ts b/src/plugins/data/public/search/aggs/metrics/sibling_pipeline.test.ts index 8389ed8262ce5..ab480fe44227e 100644 --- a/src/plugins/data/public/search/aggs/metrics/sibling_pipeline.test.ts +++ b/src/plugins/data/public/search/aggs/metrics/sibling_pipeline.test.ts @@ -17,27 +17,47 @@ * under the License. */ -import { bucketSumMetricAgg } from './bucket_sum'; -import { bucketAvgMetricAgg } from './bucket_avg'; -import { bucketMinMetricAgg } from './bucket_min'; -import { bucketMaxMetricAgg } from './bucket_max'; +import { getBucketSumMetricAgg } from './bucket_sum'; +import { getBucketAvgMetricAgg } from './bucket_avg'; +import { getBucketMinMetricAgg } from './bucket_min'; +import { getBucketMaxMetricAgg } from './bucket_max'; import { AggConfigs } from '../agg_configs'; import { IMetricAggConfig, MetricAggType } from './metric_agg_type'; -import { mockDataServices, mockAggTypesRegistry } from '../test_helpers'; +import { mockAggTypesRegistry } from '../test_helpers'; +import { fieldFormatsServiceMock } from '../../../field_formats/mocks'; +import { GetInternalStartServicesFn } from '../../../types'; +import { notificationServiceMock } from '../../../../../../../src/core/public/mocks'; describe('sibling pipeline aggs', () => { - beforeEach(() => { - mockDataServices(); + const getInternalStartServices: GetInternalStartServicesFn = () => ({ + fieldFormats: fieldFormatsServiceMock.createStartContract(), + notifications: notificationServiceMock.createStartContract(), }); const typesRegistry = mockAggTypesRegistry(); const metrics = [ - { name: 'sum_bucket', title: 'Overall Sum', provider: bucketSumMetricAgg }, - { name: 'avg_bucket', title: 'Overall Average', provider: bucketAvgMetricAgg }, - { name: 'min_bucket', title: 'Overall Min', provider: bucketMinMetricAgg }, - { name: 'max_bucket', title: 'Overall Max', provider: bucketMaxMetricAgg }, + { + name: 'sum_bucket', + title: 'Overall Sum', + provider: getBucketSumMetricAgg({ getInternalStartServices }), + }, + { + name: 'avg_bucket', + title: 'Overall Average', + provider: getBucketAvgMetricAgg({ getInternalStartServices }), + }, + { + name: 'min_bucket', + title: 'Overall Min', + provider: getBucketMinMetricAgg({ getInternalStartServices }), + }, + { + name: 'max_bucket', + title: 'Overall Max', + provider: getBucketMaxMetricAgg({ getInternalStartServices }), + }, ]; metrics.forEach(metric => { diff --git a/src/plugins/data/public/search/aggs/metrics/std_deviation.test.ts b/src/plugins/data/public/search/aggs/metrics/std_deviation.test.ts index 0679831b1e6ac..6bbff3009cc11 100644 --- a/src/plugins/data/public/search/aggs/metrics/std_deviation.test.ts +++ b/src/plugins/data/public/search/aggs/metrics/std_deviation.test.ts @@ -17,13 +17,25 @@ * under the License. */ -import { IStdDevAggConfig, stdDeviationMetricAgg } from './std_deviation'; +import { + IStdDevAggConfig, + getStdDeviationMetricAgg, + StdDeviationMetricAggDependencies, +} from './std_deviation'; import { AggConfigs } from '../agg_configs'; import { mockAggTypesRegistry } from '../test_helpers'; import { METRIC_TYPES } from './metric_agg_types'; +import { fieldFormatsServiceMock } from '../../../field_formats/mocks'; +import { notificationServiceMock } from '../../../../../../../src/core/public/mocks'; describe('AggTypeMetricStandardDeviationProvider class', () => { - const typesRegistry = mockAggTypesRegistry([stdDeviationMetricAgg]); + const aggTypesDependencies: StdDeviationMetricAggDependencies = { + getInternalStartServices: () => ({ + fieldFormats: fieldFormatsServiceMock.createStartContract(), + notifications: notificationServiceMock.createStartContract(), + }), + }; + const typesRegistry = mockAggTypesRegistry([getStdDeviationMetricAgg(aggTypesDependencies)]); const getAggConfigs = (customLabel?: string) => { const field = { name: 'memory', @@ -58,7 +70,7 @@ describe('AggTypeMetricStandardDeviationProvider class', () => { it('uses the custom label if it is set', () => { const aggConfigs = getAggConfigs('custom label'); - const responseAggs: any = stdDeviationMetricAgg.getResponseAggs( + const responseAggs: any = getStdDeviationMetricAgg(aggTypesDependencies).getResponseAggs( aggConfigs.aggs[0] as IStdDevAggConfig ); @@ -72,7 +84,7 @@ describe('AggTypeMetricStandardDeviationProvider class', () => { it('uses the default labels if custom label is not set', () => { const aggConfigs = getAggConfigs(); - const responseAggs: any = stdDeviationMetricAgg.getResponseAggs( + const responseAggs: any = getStdDeviationMetricAgg(aggTypesDependencies).getResponseAggs( aggConfigs.aggs[0] as IStdDevAggConfig ); diff --git a/src/plugins/data/public/search/aggs/metrics/std_deviation.ts b/src/plugins/data/public/search/aggs/metrics/std_deviation.ts index 5e069e317e052..e972132542ceb 100644 --- a/src/plugins/data/public/search/aggs/metrics/std_deviation.ts +++ b/src/plugins/data/public/search/aggs/metrics/std_deviation.ts @@ -23,6 +23,7 @@ import { MetricAggType } from './metric_agg_type'; import { METRIC_TYPES } from './metric_agg_types'; import { getResponseAggConfigClass, IResponseAggConfig } from './lib/get_response_agg_config_class'; import { KBN_FIELD_TYPES } from '../../../../common'; +import { GetInternalStartServicesFn } from '../../../types'; interface ValProp { valProp: string[]; @@ -34,6 +35,10 @@ export interface IStdDevAggConfig extends IResponseAggConfig { valProp: () => ValProp; } +export interface StdDeviationMetricAggDependencies { + getInternalStartServices: GetInternalStartServicesFn; +} + const responseAggConfigProps = { valProp(this: IStdDevAggConfig) { const customLabel = this.getParam('customLabel'); @@ -75,33 +80,42 @@ const responseAggConfigProps = { }, }; -export const stdDeviationMetricAgg = new MetricAggType({ - name: METRIC_TYPES.STD_DEV, - dslName: 'extended_stats', - title: i18n.translate('data.search.aggs.metrics.standardDeviationTitle', { - defaultMessage: 'Standard Deviation', - }), - makeLabel(agg) { - return i18n.translate('data.search.aggs.metrics.standardDeviationLabel', { - defaultMessage: 'Standard Deviation of {field}', - values: { field: agg.getFieldDisplayName() }, - }); - }, - params: [ +export const getStdDeviationMetricAgg = ({ + getInternalStartServices, +}: StdDeviationMetricAggDependencies) => { + return new MetricAggType( { - name: 'field', - type: 'field', - filterFieldTypes: KBN_FIELD_TYPES.NUMBER, - }, - ], + name: METRIC_TYPES.STD_DEV, + dslName: 'extended_stats', + title: i18n.translate('data.search.aggs.metrics.standardDeviationTitle', { + defaultMessage: 'Standard Deviation', + }), + makeLabel(agg) { + return i18n.translate('data.search.aggs.metrics.standardDeviationLabel', { + defaultMessage: 'Standard Deviation of {field}', + values: { field: agg.getFieldDisplayName() }, + }); + }, + params: [ + { + name: 'field', + type: 'field', + filterFieldTypes: KBN_FIELD_TYPES.NUMBER, + }, + ], - getResponseAggs(agg) { - const ValueAggConfig = getResponseAggConfigClass(agg, responseAggConfigProps); + getResponseAggs(agg) { + const ValueAggConfig = getResponseAggConfigClass(agg, responseAggConfigProps); - return [new ValueAggConfig('std_lower'), new ValueAggConfig('std_upper')]; - }, + return [new ValueAggConfig('std_lower'), new ValueAggConfig('std_upper')]; + }, - getValue(agg, bucket) { - return get(bucket[agg.parentId], agg.valProp()); - }, -}); + getValue(agg, bucket) { + return get(bucket[agg.parentId], agg.valProp()); + }, + }, + { + getInternalStartServices, + } + ); +}; diff --git a/src/plugins/data/public/search/aggs/metrics/sum.ts b/src/plugins/data/public/search/aggs/metrics/sum.ts index ffb117dda0839..545c6d6a4939e 100644 --- a/src/plugins/data/public/search/aggs/metrics/sum.ts +++ b/src/plugins/data/public/search/aggs/metrics/sum.ts @@ -21,28 +21,40 @@ import { i18n } from '@kbn/i18n'; import { MetricAggType } from './metric_agg_type'; import { METRIC_TYPES } from './metric_agg_types'; import { KBN_FIELD_TYPES } from '../../../../common'; +import { GetInternalStartServicesFn } from '../../../types'; const sumTitle = i18n.translate('data.search.aggs.metrics.sumTitle', { defaultMessage: 'Sum', }); -export const sumMetricAgg = new MetricAggType({ - name: METRIC_TYPES.SUM, - title: sumTitle, - makeLabel(aggConfig) { - return i18n.translate('data.search.aggs.metrics.sumLabel', { - defaultMessage: 'Sum of {field}', - values: { field: aggConfig.getFieldDisplayName() }, - }); - }, - isScalable() { - return true; - }, - params: [ +export interface SumMetricAggDependencies { + getInternalStartServices: GetInternalStartServicesFn; +} + +export const getSumMetricAgg = ({ getInternalStartServices }: SumMetricAggDependencies) => { + return new MetricAggType( { - name: 'field', - type: 'field', - filterFieldTypes: KBN_FIELD_TYPES.NUMBER, + name: METRIC_TYPES.SUM, + title: sumTitle, + makeLabel(aggConfig) { + return i18n.translate('data.search.aggs.metrics.sumLabel', { + defaultMessage: 'Sum of {field}', + values: { field: aggConfig.getFieldDisplayName() }, + }); + }, + isScalable() { + return true; + }, + params: [ + { + name: 'field', + type: 'field', + filterFieldTypes: KBN_FIELD_TYPES.NUMBER, + }, + ], }, - ], -}); + { + getInternalStartServices, + } + ); +}; diff --git a/src/plugins/data/public/search/aggs/metrics/top_hit.test.ts b/src/plugins/data/public/search/aggs/metrics/top_hit.test.ts index c65a714f26247..8294ad09bae22 100644 --- a/src/plugins/data/public/search/aggs/metrics/top_hit.test.ts +++ b/src/plugins/data/public/search/aggs/metrics/top_hit.test.ts @@ -18,15 +18,23 @@ */ import { dropRight, last } from 'lodash'; -import { topHitMetricAgg } from './top_hit'; +import { getTopHitMetricAgg, TopHitMetricAggDependencies } from './top_hit'; import { AggConfigs } from '../agg_configs'; import { mockAggTypesRegistry } from '../test_helpers'; import { IMetricAggConfig } from './metric_agg_type'; import { KBN_FIELD_TYPES } from '../../../../common'; +import { fieldFormatsServiceMock } from '../../../field_formats/mocks'; +import { notificationServiceMock } from '../../../../../../../src/core/public/mocks'; describe('Top hit metric', () => { let aggDsl: Record; let aggConfig: IMetricAggConfig; + const aggTypesDependencies: TopHitMetricAggDependencies = { + getInternalStartServices: () => ({ + fieldFormats: fieldFormatsServiceMock.createStartContract(), + notifications: notificationServiceMock.createStartContract(), + }), + }; const init = ({ fieldName = 'field', @@ -36,7 +44,7 @@ describe('Top hit metric', () => { fieldType = KBN_FIELD_TYPES.NUMBER, size = 1, }: any) => { - const typesRegistry = mockAggTypesRegistry([topHitMetricAgg]); + const typesRegistry = mockAggTypesRegistry([getTopHitMetricAgg(aggTypesDependencies)]); const field = { name: fieldName, displayName: fieldName, @@ -91,7 +99,7 @@ describe('Top hit metric', () => { it('should return a label prefixed with Last if sorting in descending order', () => { init({ fieldName: 'bytes' }); - expect(topHitMetricAgg.makeLabel(aggConfig)).toEqual('Last bytes'); + expect(getTopHitMetricAgg(aggTypesDependencies).makeLabel(aggConfig)).toEqual('Last bytes'); }); it('should return a label prefixed with First if sorting in ascending order', () => { @@ -99,7 +107,7 @@ describe('Top hit metric', () => { fieldName: 'bytes', sortOrder: 'asc', }); - expect(topHitMetricAgg.makeLabel(aggConfig)).toEqual('First bytes'); + expect(getTopHitMetricAgg(aggTypesDependencies).makeLabel(aggConfig)).toEqual('First bytes'); }); it('should request the _source field', () => { @@ -140,7 +148,7 @@ describe('Top hit metric', () => { }; init({ fieldName: '@tags' }); - expect(topHitMetricAgg.getValue(aggConfig, bucket)).toBe(null); + expect(getTopHitMetricAgg(aggTypesDependencies).getValue(aggConfig, bucket)).toBe(null); }); // it('should return undefined if the field does not appear in the source', () => { @@ -159,7 +167,7 @@ describe('Top hit metric', () => { }; init({ fieldName: '@tags' }); - expect(topHitMetricAgg.getValue(aggConfig, bucket)).toBe(undefined); + expect(getTopHitMetricAgg(aggTypesDependencies).getValue(aggConfig, bucket)).toBe(undefined); }); it('should return the field value from the top hit', () => { @@ -178,7 +186,7 @@ describe('Top hit metric', () => { }; init({ fieldName: '@tags' }); - expect(topHitMetricAgg.getValue(aggConfig, bucket)).toBe('aaa'); + expect(getTopHitMetricAgg(aggTypesDependencies).getValue(aggConfig, bucket)).toBe('aaa'); }); it('should return the object if the field value is an object', () => { @@ -200,7 +208,9 @@ describe('Top hit metric', () => { init({ fieldName: '@tags' }); - expect(topHitMetricAgg.getValue(aggConfig, bucket)).toEqual({ label: 'aaa' }); + expect(getTopHitMetricAgg(aggTypesDependencies).getValue(aggConfig, bucket)).toEqual({ + label: 'aaa', + }); }); it('should return an array if the field has more than one values', () => { @@ -219,7 +229,10 @@ describe('Top hit metric', () => { }; init({ fieldName: '@tags' }); - expect(topHitMetricAgg.getValue(aggConfig, bucket)).toEqual(['aaa', 'bbb']); + expect(getTopHitMetricAgg(aggTypesDependencies).getValue(aggConfig, bucket)).toEqual([ + 'aaa', + 'bbb', + ]); }); it('should return undefined if the field is not in the source nor in the doc_values field', () => { @@ -241,7 +254,7 @@ describe('Top hit metric', () => { }; init({ fieldName: 'machine.os.raw', readFromDocValues: true }); - expect(topHitMetricAgg.getValue(aggConfig, bucket)).toBe(undefined); + expect(getTopHitMetricAgg(aggTypesDependencies).getValue(aggConfig, bucket)).toBe(undefined); }); describe('Multivalued field and first/last X docs', () => { @@ -250,7 +263,9 @@ describe('Top hit metric', () => { fieldName: 'bytes', size: 2, }); - expect(topHitMetricAgg.makeLabel(aggConfig)).toEqual('Last 2 bytes'); + expect(getTopHitMetricAgg(aggTypesDependencies).makeLabel(aggConfig)).toEqual( + 'Last 2 bytes' + ); }); it('should return a label prefixed with First X docs if sorting in ascending order', () => { @@ -259,7 +274,9 @@ describe('Top hit metric', () => { size: 2, sortOrder: 'asc', }); - expect(topHitMetricAgg.makeLabel(aggConfig)).toEqual('First 2 bytes'); + expect(getTopHitMetricAgg(aggTypesDependencies).makeLabel(aggConfig)).toEqual( + 'First 2 bytes' + ); }); [ @@ -334,7 +351,9 @@ describe('Top hit metric', () => { }; init({ fieldName: 'bytes', aggregate: agg.type }); - expect(topHitMetricAgg.getValue(aggConfig, bucket)).toEqual(agg.result); + expect(getTopHitMetricAgg(aggTypesDependencies).getValue(aggConfig, bucket)).toEqual( + agg.result + ); }); it(`should return the result of the ${agg.type} aggregation over the last X docs - ${agg.description}`, () => { @@ -358,7 +377,9 @@ describe('Top hit metric', () => { }; init({ fieldName: 'bytes', aggregate: agg.type }); - expect(topHitMetricAgg.getValue(aggConfig, bucket)).toEqual(agg.result); + expect(getTopHitMetricAgg(aggTypesDependencies).getValue(aggConfig, bucket)).toEqual( + agg.result + ); }); }); }); diff --git a/src/plugins/data/public/search/aggs/metrics/top_hit.ts b/src/plugins/data/public/search/aggs/metrics/top_hit.ts index d0c668c577e62..15da2b485aee7 100644 --- a/src/plugins/data/public/search/aggs/metrics/top_hit.ts +++ b/src/plugins/data/public/search/aggs/metrics/top_hit.ts @@ -22,6 +22,11 @@ import { i18n } from '@kbn/i18n'; import { IMetricAggConfig, MetricAggType } from './metric_agg_type'; import { METRIC_TYPES } from './metric_agg_types'; import { KBN_FIELD_TYPES } from '../../../../common'; +import { GetInternalStartServicesFn } from '../../../types'; + +export interface TopHitMetricAggDependencies { + getInternalStartServices: GetInternalStartServicesFn; +} const isNumericFieldSelected = (agg: IMetricAggConfig) => { const field = agg.getParam('field'); @@ -29,214 +34,225 @@ const isNumericFieldSelected = (agg: IMetricAggConfig) => { return field && field.type && field.type === KBN_FIELD_TYPES.NUMBER; }; -export const topHitMetricAgg = new MetricAggType({ - name: METRIC_TYPES.TOP_HITS, - title: i18n.translate('data.search.aggs.metrics.topHitTitle', { - defaultMessage: 'Top Hit', - }), - makeLabel(aggConfig) { - const lastPrefixLabel = i18n.translate('data.search.aggs.metrics.topHit.lastPrefixLabel', { - defaultMessage: 'Last', - }); - const firstPrefixLabel = i18n.translate('data.search.aggs.metrics.topHit.firstPrefixLabel', { - defaultMessage: 'First', - }); - - let prefix = - aggConfig.getParam('sortOrder').value === 'desc' ? lastPrefixLabel : firstPrefixLabel; - - const size = aggConfig.getParam('size'); - - if (size !== 1) { - prefix += ` ${size}`; - } - - const field = aggConfig.getParam('field'); - - return `${prefix} ${field ? field.displayName : ''}`; - }, - params: [ +export const getTopHitMetricAgg = ({ getInternalStartServices }: TopHitMetricAggDependencies) => { + return new MetricAggType( { - name: 'field', - type: 'field', - onlyAggregatable: false, - filterFieldTypes: Object.values(KBN_FIELD_TYPES).filter( - type => type !== KBN_FIELD_TYPES.HISTOGRAM - ), - write(agg, output) { - const field = agg.getParam('field'); - output.params = {}; - - if (field.scripted) { - output.params.script_fields = { - [field.name]: { - script: { - source: field.script, - lang: field.lang, - }, - }, - }; - } else { - if (field.readFromDocValues) { - // always format date fields as date_time to avoid - // displaying unformatted dates like epoch_millis - // or other not-accepted momentjs formats - const format = field.type === KBN_FIELD_TYPES.DATE ? 'date_time' : 'use_field_mapping'; - output.params.docvalue_fields = [{ field: field.name, format }]; + name: METRIC_TYPES.TOP_HITS, + title: i18n.translate('data.search.aggs.metrics.topHitTitle', { + defaultMessage: 'Top Hit', + }), + makeLabel(aggConfig) { + const lastPrefixLabel = i18n.translate('data.search.aggs.metrics.topHit.lastPrefixLabel', { + defaultMessage: 'Last', + }); + const firstPrefixLabel = i18n.translate( + 'data.search.aggs.metrics.topHit.firstPrefixLabel', + { + defaultMessage: 'First', } - output.params._source = field.name === '_source' ? true : field.name; + ); + + let prefix = + aggConfig.getParam('sortOrder').value === 'desc' ? lastPrefixLabel : firstPrefixLabel; + + const size = aggConfig.getParam('size'); + + if (size !== 1) { + prefix += ` ${size}`; } + + const field = aggConfig.getParam('field'); + + return `${prefix} ${field ? field.displayName : ''}`; }, - }, - { - name: 'aggregate', - type: 'optioned', - options: [ + params: [ { - text: i18n.translate('data.search.aggs.metrics.topHit.minLabel', { - defaultMessage: 'Min', - }), - isCompatible: isNumericFieldSelected, - disabled: true, - value: 'min', - }, - { - text: i18n.translate('data.search.aggs.metrics.topHit.maxLabel', { - defaultMessage: 'Max', - }), - isCompatible: isNumericFieldSelected, - disabled: true, - value: 'max', + name: 'field', + type: 'field', + onlyAggregatable: false, + filterFieldTypes: Object.values(KBN_FIELD_TYPES).filter( + type => type !== KBN_FIELD_TYPES.HISTOGRAM + ), + write(agg, output) { + const field = agg.getParam('field'); + output.params = {}; + + if (field.scripted) { + output.params.script_fields = { + [field.name]: { + script: { + source: field.script, + lang: field.lang, + }, + }, + }; + } else { + if (field.readFromDocValues) { + // always format date fields as date_time to avoid + // displaying unformatted dates like epoch_millis + // or other not-accepted momentjs formats + const format = + field.type === KBN_FIELD_TYPES.DATE ? 'date_time' : 'use_field_mapping'; + output.params.docvalue_fields = [{ field: field.name, format }]; + } + output.params._source = field.name === '_source' ? true : field.name; + } + }, }, { - text: i18n.translate('data.search.aggs.metrics.topHit.sumLabel', { - defaultMessage: 'Sum', - }), - isCompatible: isNumericFieldSelected, - disabled: true, - value: 'sum', + name: 'aggregate', + type: 'optioned', + options: [ + { + text: i18n.translate('data.search.aggs.metrics.topHit.minLabel', { + defaultMessage: 'Min', + }), + isCompatible: isNumericFieldSelected, + disabled: true, + value: 'min', + }, + { + text: i18n.translate('data.search.aggs.metrics.topHit.maxLabel', { + defaultMessage: 'Max', + }), + isCompatible: isNumericFieldSelected, + disabled: true, + value: 'max', + }, + { + text: i18n.translate('data.search.aggs.metrics.topHit.sumLabel', { + defaultMessage: 'Sum', + }), + isCompatible: isNumericFieldSelected, + disabled: true, + value: 'sum', + }, + { + text: i18n.translate('data.search.aggs.metrics.topHit.averageLabel', { + defaultMessage: 'Average', + }), + isCompatible: isNumericFieldSelected, + disabled: true, + value: 'average', + }, + { + text: i18n.translate('data.search.aggs.metrics.topHit.concatenateLabel', { + defaultMessage: 'Concatenate', + }), + isCompatible(aggConfig: IMetricAggConfig) { + return _.get(aggConfig.params, 'field.filterFieldTypes', '*') === '*'; + }, + disabled: true, + value: 'concat', + }, + ], + write: _.noop, }, { - text: i18n.translate('data.search.aggs.metrics.topHit.averageLabel', { - defaultMessage: 'Average', - }), - isCompatible: isNumericFieldSelected, - disabled: true, - value: 'average', + name: 'size', + default: 1, }, { - text: i18n.translate('data.search.aggs.metrics.topHit.concatenateLabel', { - defaultMessage: 'Concatenate', - }), - isCompatible(aggConfig: IMetricAggConfig) { - return _.get(aggConfig.params, 'field.filterFieldTypes', '*') === '*'; + name: 'sortField', + type: 'field', + filterFieldTypes: [ + KBN_FIELD_TYPES.NUMBER, + KBN_FIELD_TYPES.DATE, + KBN_FIELD_TYPES.IP, + KBN_FIELD_TYPES.STRING, + ], + default(agg: IMetricAggConfig) { + return agg.getIndexPattern().timeFieldName; }, - disabled: true, - value: 'concat', - }, - ], - write: _.noop, - }, - { - name: 'size', - default: 1, - }, - { - name: 'sortField', - type: 'field', - filterFieldTypes: [ - KBN_FIELD_TYPES.NUMBER, - KBN_FIELD_TYPES.DATE, - KBN_FIELD_TYPES.IP, - KBN_FIELD_TYPES.STRING, - ], - default(agg: IMetricAggConfig) { - return agg.getIndexPattern().timeFieldName; - }, - write: _.noop, // prevent default write, it is handled below - }, - { - name: 'sortOrder', - type: 'optioned', - default: 'desc', - options: [ - { - text: i18n.translate('data.search.aggs.metrics.topHit.descendingLabel', { - defaultMessage: 'Descending', - }), - value: 'desc', + write: _.noop, // prevent default write, it is handled below }, { - text: i18n.translate('data.search.aggs.metrics.topHit.ascendingLabel', { - defaultMessage: 'Ascending', - }), - value: 'asc', - }, - ], - write(agg, output) { - const sortField = agg.params.sortField; - const sortOrder = agg.params.sortOrder; - - if (sortField.scripted) { - output.params.sort = [ + name: 'sortOrder', + type: 'optioned', + default: 'desc', + options: [ { - _script: { - script: { - source: sortField.script, - lang: sortField.lang, - }, - type: sortField.type, - order: sortOrder.value, - }, + text: i18n.translate('data.search.aggs.metrics.topHit.descendingLabel', { + defaultMessage: 'Descending', + }), + value: 'desc', }, - ]; - } else { - output.params.sort = [ { - [sortField.name]: { - order: sortOrder.value, - }, + text: i18n.translate('data.search.aggs.metrics.topHit.ascendingLabel', { + defaultMessage: 'Ascending', + }), + value: 'asc', }, - ]; + ], + write(agg, output) { + const sortField = agg.params.sortField; + const sortOrder = agg.params.sortOrder; + + if (sortField.scripted) { + output.params.sort = [ + { + _script: { + script: { + source: sortField.script, + lang: sortField.lang, + }, + type: sortField.type, + order: sortOrder.value, + }, + }, + ]; + } else { + output.params.sort = [ + { + [sortField.name]: { + order: sortOrder.value, + }, + }, + ]; + } + }, + }, + ], + getValue(agg, bucket) { + const hits: any[] = _.get(bucket, `${agg.id}.hits.hits`); + if (!hits || !hits.length) { + return null; } - }, - }, - ], - getValue(agg, bucket) { - const hits: any[] = _.get(bucket, `${agg.id}.hits.hits`); - if (!hits || !hits.length) { - return null; - } - const path = agg.getParam('field').name; + const path = agg.getParam('field').name; - let values = _.flatten( - hits.map(hit => - path === '_source' ? hit._source : agg.getIndexPattern().flattenHit(hit, true)[path] - ) - ); + let values = _.flatten( + hits.map(hit => + path === '_source' ? hit._source : agg.getIndexPattern().flattenHit(hit, true)[path] + ) + ); - if (values.length === 1) { - values = values[0]; - } + if (values.length === 1) { + values = values[0]; + } + + if (Array.isArray(values)) { + if (!_.compact(values).length) { + return null; + } + + const aggregate = agg.getParam('aggregate'); - if (Array.isArray(values)) { - if (!_.compact(values).length) { - return null; - } - - const aggregate = agg.getParam('aggregate'); - - switch (aggregate.value) { - case 'max': - return _.max(values); - case 'min': - return _.min(values); - case 'sum': - return _.sum(values); - case 'average': - return _.sum(values) / values.length; - } + switch (aggregate.value) { + case 'max': + return _.max(values); + case 'min': + return _.min(values); + case 'sum': + return _.sum(values); + case 'average': + return _.sum(values) / values.length; + } + } + return values; + }, + }, + { + getInternalStartServices, } - return values; - }, -}); + ); +}; diff --git a/src/plugins/data/public/search/aggs/param_types/field.test.ts b/src/plugins/data/public/search/aggs/param_types/field.test.ts index 0182471392910..ea7931130b84a 100644 --- a/src/plugins/data/public/search/aggs/param_types/field.test.ts +++ b/src/plugins/data/public/search/aggs/param_types/field.test.ts @@ -18,11 +18,20 @@ */ import { BaseParamType } from './base'; -import { FieldParamType } from './field'; +import { FieldParamType, FieldParamTypeDependencies } from './field'; import { ES_FIELD_TYPES, KBN_FIELD_TYPES } from '../../../../common'; import { IAggConfig } from '../agg_config'; +import { fieldFormatsServiceMock } from '../../../field_formats/mocks'; +import { notificationServiceMock } from '../../../../../../../src/core/public/mocks'; describe('Field', () => { + const fieldParamTypeDependencies: FieldParamTypeDependencies = { + getInternalStartServices: () => ({ + fieldFormats: fieldFormatsServiceMock.createStartContract(), + notifications: notificationServiceMock.createStartContract(), + }), + }; + const indexPattern = { id: '1234', title: 'logstash-*', @@ -52,10 +61,13 @@ describe('Field', () => { describe('constructor', () => { it('it is an instance of BaseParamType', () => { - const aggParam = new FieldParamType({ - name: 'field', - type: 'field', - }); + const aggParam = new FieldParamType( + { + name: 'field', + type: 'field', + }, + fieldParamTypeDependencies + ); expect(aggParam instanceof BaseParamType).toBeTruthy(); }); @@ -63,10 +75,13 @@ describe('Field', () => { describe('getAvailableFields', () => { it('should return only aggregatable fields by default', () => { - const aggParam = new FieldParamType({ - name: 'field', - type: 'field', - }); + const aggParam = new FieldParamType( + { + name: 'field', + type: 'field', + }, + fieldParamTypeDependencies + ); const fields = aggParam.getAvailableFields(agg); @@ -78,10 +93,13 @@ describe('Field', () => { }); it('should return all fields if onlyAggregatable is false', () => { - const aggParam = new FieldParamType({ - name: 'field', - type: 'field', - }); + const aggParam = new FieldParamType( + { + name: 'field', + type: 'field', + }, + fieldParamTypeDependencies + ); aggParam.onlyAggregatable = false; @@ -91,10 +109,13 @@ describe('Field', () => { }); it('should return all fields if filterFieldTypes was not specified', () => { - const aggParam = new FieldParamType({ - name: 'field', - type: 'field', - }); + const aggParam = new FieldParamType( + { + name: 'field', + type: 'field', + }, + fieldParamTypeDependencies + ); indexPattern.fields[1].aggregatable = true; diff --git a/src/plugins/data/public/search/aggs/param_types/field.ts b/src/plugins/data/public/search/aggs/param_types/field.ts index 34b77e14a3a71..4d67f41905c5a 100644 --- a/src/plugins/data/public/search/aggs/param_types/field.ts +++ b/src/plugins/data/public/search/aggs/param_types/field.ts @@ -24,7 +24,7 @@ import { BaseParamType } from './base'; import { propFilter } from '../filter'; import { isNestedField, KBN_FIELD_TYPES } from '../../../../common'; import { Field as IndexPatternField } from '../../../index_patterns'; -import { getNotifications } from '../../../../public/services'; +import { GetInternalStartServicesFn } from '../../../types'; const filterByType = propFilter('type'); @@ -32,13 +32,20 @@ export type FieldTypes = KBN_FIELD_TYPES | KBN_FIELD_TYPES[] | '*'; // TODO need to make a more explicit interface for this export type IFieldParamType = FieldParamType; +export interface FieldParamTypeDependencies { + getInternalStartServices: GetInternalStartServicesFn; +} + export class FieldParamType extends BaseParamType { required = true; scriptable = true; filterFieldTypes: FieldTypes; onlyAggregatable: boolean; - constructor(config: Record) { + constructor( + config: Record, + { getInternalStartServices }: FieldParamTypeDependencies + ) { super(config); this.filterFieldTypes = config.filterFieldTypes || '*'; @@ -87,7 +94,7 @@ export class FieldParamType extends BaseParamType { // @ts-ignore const validField = this.getAvailableFields(aggConfig).find((f: any) => f.name === fieldName); if (!validField) { - getNotifications().toasts.addDanger( + getInternalStartServices().notifications.toasts.addDanger( i18n.translate( 'data.search.aggs.paramTypes.field.invalidSavedFieldParameterErrorMessage', { diff --git a/src/plugins/data/public/search/aggs/test_helpers/mock_agg_types_registry.ts b/src/plugins/data/public/search/aggs/test_helpers/mock_agg_types_registry.ts index 57d27b7da6313..2383affa2a8c5 100644 --- a/src/plugins/data/public/search/aggs/test_helpers/mock_agg_types_registry.ts +++ b/src/plugins/data/public/search/aggs/test_helpers/mock_agg_types_registry.ts @@ -18,12 +18,13 @@ */ // eslint-disable-next-line @kbn/eslint/no-restricted-paths -import { coreMock } from '../../../../../../../src/core/public/mocks'; +import { coreMock, notificationServiceMock } from '../../../../../../../src/core/public/mocks'; import { AggTypesRegistry, AggTypesRegistryStart } from '../agg_types_registry'; import { getAggTypes } from '../agg_types'; -import { BucketAggType } from '../buckets/_bucket_agg_type'; +import { BucketAggType } from '../buckets/bucket_agg_type'; import { MetricAggType } from '../metrics/metric_agg_type'; import { queryServiceMock } from '../../../query/mocks'; +import { fieldFormatsServiceMock } from '../../../field_formats/mocks'; /** * Testing utility which creates a new instance of AggTypesRegistry, @@ -55,8 +56,11 @@ export function mockAggTypesRegistry | MetricAggTyp const core = coreMock.createSetup(); const aggTypes = getAggTypes({ uiSettings: core.uiSettings, - notifications: core.notifications, query: queryServiceMock.createSetupContract(), + getInternalStartServices: () => ({ + fieldFormats: fieldFormatsServiceMock.createStartContract(), + notifications: notificationServiceMock.createStartContract(), + }), }); aggTypes.buckets.forEach(type => registrySetup.registerBucket(type)); diff --git a/src/plugins/data/public/search/search_service.ts b/src/plugins/data/public/search/search_service.ts index dc1c99f76d59a..42f31ef450d28 100644 --- a/src/plugins/data/public/search/search_service.ts +++ b/src/plugins/data/public/search/search_service.ts @@ -26,6 +26,7 @@ import { getEsClient, LegacyApiCaller } from './es_client'; import { ES_SEARCH_STRATEGY, DEFAULT_SEARCH_STRATEGY } from '../../common/search'; import { esSearchStrategyProvider } from './es_search/es_search_strategy'; import { QuerySetup } from '../query/query_service'; +import { GetInternalStartServicesFn } from '../types'; import { SearchInterceptor } from './search_interceptor'; import { getAggTypes, @@ -44,6 +45,7 @@ import { interface SearchServiceSetupDependencies { packageInfo: PackageInfo; query: QuerySetup; + getInternalStartServices: GetInternalStartServicesFn; } /** @@ -81,7 +83,7 @@ export class SearchService implements Plugin { public setup( core: CoreSetup, - { packageInfo, query }: SearchServiceSetupDependencies + { packageInfo, query, getInternalStartServices }: SearchServiceSetupDependencies ): ISearchSetup { this.esClient = getEsClient(core.injectedMetadata, core.http, packageInfo); this.registerSearchStrategyProvider(SYNC_SEARCH_STRATEGY, syncSearchStrategyProvider); @@ -91,7 +93,7 @@ export class SearchService implements Plugin { const aggTypes = getAggTypes({ query, uiSettings: core.uiSettings, - notifications: core.notifications, + getInternalStartServices, }); aggTypes.buckets.forEach(b => aggTypesSetup.registerBucket(b)); diff --git a/src/plugins/data/public/types.ts b/src/plugins/data/public/types.ts index 45160cbf30179..e24e01d241278 100644 --- a/src/plugins/data/public/types.ts +++ b/src/plugins/data/public/types.ts @@ -71,3 +71,12 @@ export interface IDataPluginServices extends Partial { storage: IStorageWrapper; data: DataPublicPluginStart; } + +/** @internal **/ +export interface InternalStartServices { + fieldFormats: FieldFormatsStart; + notifications: CoreStart['notifications']; +} + +/** @internal **/ +export type GetInternalStartServicesFn = () => InternalStartServices; diff --git a/src/plugins/data/server/index_patterns/routes.ts b/src/plugins/data/server/index_patterns/routes.ts index 8f017a73083ec..8b9fa28c77165 100644 --- a/src/plugins/data/server/index_patterns/routes.ts +++ b/src/plugins/data/server/index_patterns/routes.ts @@ -70,7 +70,22 @@ export function registerRoutes(http: HttpServiceSetup) { }, }); } catch (error) { - return response.notFound(); + if ( + typeof error === 'object' && + !!error?.isBoom && + !!error?.output?.payload && + typeof error?.output?.payload === 'object' + ) { + const payload = error?.output?.payload; + return response.notFound({ + body: { + message: payload.message, + attributes: payload, + }, + }); + } else { + return response.notFound(); + } } } ); diff --git a/test/functional/apps/discover/_discover.js b/test/functional/apps/discover/_discover.js index 850b2773b5025..4f357e2993b30 100644 --- a/test/functional/apps/discover/_discover.js +++ b/test/functional/apps/discover/_discover.js @@ -44,7 +44,6 @@ export default function({ getService, getPageObjects }) { }); describe('query', function() { - this.tags(['skipFirefox']); const queryName1 = 'Query # 1'; it('should show correct time range string by timepicker', async function() { @@ -100,9 +99,10 @@ export default function({ getService, getPageObjects }) { const newDurationHours = await PageObjects.timePicker.getTimeDurationInHours(); expect(Math.round(newDurationHours)).to.be(25); const rowData = await PageObjects.discover.getDocTableField(1); + log.debug(`The first timestamp value in doc table: ${rowData}`); expect(Date.parse(rowData)).to.be.within( - Date.parse('Sep 20, 2015 @ 22:00:00.000'), - Date.parse('Sep 20, 2015 @ 23:30:00.000') + Date.parse('Sep 20, 2015 @ 21:30:00.000'), + Date.parse('Sep 20, 2015 @ 23:00:00.000') ); }); diff --git a/test/functional/page_objects/common_page.ts b/test/functional/page_objects/common_page.ts index de4917ef2b1b3..f06baeb7a4167 100644 --- a/test/functional/page_objects/common_page.ts +++ b/test/functional/page_objects/common_page.ts @@ -43,42 +43,10 @@ export function CommonPageProvider({ getService, getPageObjects }: FtrProviderCo appConfig: {}; ensureCurrentUrl: boolean; shouldLoginIfPrompted: boolean; - shouldAcceptAlert: boolean; useActualUrl: boolean; } class CommonPage { - /** - * Navigates the browser window to provided URL - * @param url URL - * @param shouldAcceptAlert pass 'true' if browser alert should be accepted - */ - private static async navigateToUrlAndHandleAlert(url: string, shouldAcceptAlert: boolean) { - log.debug('Navigate to: ' + url); - try { - await browser.get(url); - } catch (navigationError) { - log.debug('Error navigating to url'); - const alert = await browser.getAlert(); - if (alert && alert.accept) { - if (shouldAcceptAlert) { - log.debug('Should accept alert'); - try { - await alert.accept(); - } catch (alertException) { - log.debug('Error accepting alert'); - throw alertException; - } - } else { - log.debug('Will not accept alert'); - throw navigationError; - } - } else { - throw navigationError; - } - } - } - /** * Returns Kibana host URL */ @@ -127,13 +95,7 @@ export function CommonPageProvider({ getService, getPageObjects }: FtrProviderCo } private async navigate(navigateProps: NavigateProps) { - const { - appConfig, - ensureCurrentUrl, - shouldLoginIfPrompted, - shouldAcceptAlert, - useActualUrl, - } = navigateProps; + const { appConfig, ensureCurrentUrl, shouldLoginIfPrompted, useActualUrl } = navigateProps; const appUrl = getUrl.noAuth(config.get('servers.kibana'), appConfig); await retry.try(async () => { @@ -141,7 +103,11 @@ export function CommonPageProvider({ getService, getPageObjects }: FtrProviderCo log.debug(`navigateToActualUrl ${appUrl}`); await browser.get(appUrl); } else { - await CommonPage.navigateToUrlAndHandleAlert(appUrl, shouldAcceptAlert); + log.debug(`navigateToUrl ${appUrl}`); + await browser.get(appUrl); + // accept alert if it pops up + const alert = await browser.getAlert(); + await alert?.accept(); } const currentUrl = shouldLoginIfPrompted @@ -167,7 +133,6 @@ export function CommonPageProvider({ getService, getPageObjects }: FtrProviderCo basePath = '', ensureCurrentUrl = true, shouldLoginIfPrompted = true, - shouldAcceptAlert = true, useActualUrl = false, } = {} ) { @@ -180,7 +145,6 @@ export function CommonPageProvider({ getService, getPageObjects }: FtrProviderCo appConfig, ensureCurrentUrl, shouldLoginIfPrompted, - shouldAcceptAlert, useActualUrl, }); } @@ -200,7 +164,6 @@ export function CommonPageProvider({ getService, getPageObjects }: FtrProviderCo basePath = '', ensureCurrentUrl = true, shouldLoginIfPrompted = true, - shouldAcceptAlert = true, useActualUrl = true, } = {} ) { @@ -214,7 +177,6 @@ export function CommonPageProvider({ getService, getPageObjects }: FtrProviderCo appConfig, ensureCurrentUrl, shouldLoginIfPrompted, - shouldAcceptAlert, useActualUrl, }); } @@ -228,18 +190,12 @@ export function CommonPageProvider({ getService, getPageObjects }: FtrProviderCo async navigateToActualUrl( appName: string, hash?: string, - { - basePath = '', - ensureCurrentUrl = true, - shouldLoginIfPrompted = true, - shouldAcceptAlert = true, - } = {} + { basePath = '', ensureCurrentUrl = true, shouldLoginIfPrompted = true } = {} ) { await this.navigateToUrl(appName, hash, { basePath, ensureCurrentUrl, shouldLoginIfPrompted, - shouldAcceptAlert, useActualUrl: true, }); } @@ -252,7 +208,7 @@ export function CommonPageProvider({ getService, getPageObjects }: FtrProviderCo async navigateToApp( appName: string, - { basePath = '', shouldLoginIfPrompted = true, shouldAcceptAlert = true, hash = '' } = {} + { basePath = '', shouldLoginIfPrompted = true, hash = '' } = {} ) { let appUrl: string; if (config.has(['apps', appName])) { @@ -274,7 +230,11 @@ export function CommonPageProvider({ getService, getPageObjects }: FtrProviderCo await retry.tryForTime(defaultTryTimeout * 2, async () => { let lastUrl = await retry.try(async () => { // since we're using hash URLs, always reload first to force re-render - await CommonPage.navigateToUrlAndHandleAlert(appUrl, shouldAcceptAlert); + log.debug('navigate to: ' + appUrl); + await browser.get(appUrl); + // accept alert if it pops up + const alert = await browser.getAlert(); + await alert?.accept(); await this.sleep(700); log.debug('returned from get, calling refresh'); await browser.refresh(); diff --git a/test/functional/page_objects/discover_page.ts b/test/functional/page_objects/discover_page.ts index 2377c32a80b5b..00bf87621864a 100644 --- a/test/functional/page_objects/discover_page.ts +++ b/test/functional/page_objects/discover_page.ts @@ -138,7 +138,7 @@ export function DiscoverPageProvider({ getService, getPageObjects }: FtrProvider await browser .getActions() - .move({ x: 200, y: 20, origin: el._webElement }) + .move({ x: 0, y: 20, origin: el._webElement }) .click() .perform(); } @@ -147,8 +147,8 @@ export function DiscoverPageProvider({ getService, getPageObjects }: FtrProvider const el = await elasticChart.getCanvas(); await browser.dragAndDrop( - { location: el, offset: { x: 200, y: 20 } }, - { location: el, offset: { x: 400, y: 30 } } + { location: el, offset: { x: -300, y: 20 } }, + { location: el, offset: { x: -100, y: 30 } } ); } diff --git a/test/functional/services/remote/webdriver.ts b/test/functional/services/remote/webdriver.ts index 382543822d8ac..3bf5b865aa7ba 100644 --- a/test/functional/services/remote/webdriver.ts +++ b/test/functional/services/remote/webdriver.ts @@ -107,9 +107,10 @@ async function attemptToCreateCommand( chromeOptions.push('headless', 'disable-gpu', 'remote-debugging-port=9222'); } chromeCapabilities.set('goog:chromeOptions', { - w3c: false, + w3c: true, args: chromeOptions, }); + chromeCapabilities.set('unexpectedAlertBehaviour', 'accept'); chromeCapabilities.set('goog:loggingPrefs', { browser: 'ALL' }); const session = await new Builder() diff --git a/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/Cytoscape.stories.tsx b/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/Cytoscape.stories.tsx index 46754c8c7cb6b..a8d3b843a1f3d 100644 --- a/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/Cytoscape.stories.tsx +++ b/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/Cytoscape.stories.tsx @@ -75,27 +75,17 @@ storiesOf('app/ServiceMap/Cytoscape', module) const cy = cytoscape(); const elements = [ { data: { id: 'default' } }, - { data: { id: 'cache', label: 'cache', 'span.type': 'cache' } }, - { data: { id: 'database', label: 'database', 'span.type': 'db' } }, + { data: { id: 'cache', 'span.type': 'cache' } }, + { data: { id: 'database', 'span.type': 'db' } }, { data: { id: 'elasticsearch', - label: 'elasticsearch', 'span.type': 'db', 'span.subtype': 'elasticsearch' } }, - { - data: { id: 'external', label: 'external', 'span.type': 'external' } - }, - { - data: { - id: 'messaging', - label: 'messaging', - 'span.type': 'messaging' - } - }, - + { data: { id: 'external', 'span.type': 'external' } }, + { data: { id: 'messaging', 'span.type': 'messaging' } }, { data: { id: 'dotnet', @@ -119,11 +109,18 @@ storiesOf('app/ServiceMap/Cytoscape', module) }, { data: { - id: 'js-base', - 'service.name': 'js-base service', + id: 'RUM (js-base)', + 'service.name': 'RUM service', 'agent.name': 'js-base' } }, + { + data: { + id: 'RUM (rum-js)', + 'service.name': 'RUM service', + 'agent.name': 'rum-js' + } + }, { data: { id: 'nodejs', @@ -163,7 +160,8 @@ storiesOf('app/ServiceMap/Cytoscape', module) description={ 
                     agent.name: {node.data('agent.name') || 'undefined'},
-                    span.type: {node.data('span.type') || 'undefined'}
+                    span.type: {node.data('span.type') || 'undefined'},
+                    span.subtype: {node.data('span.subtype') || 'undefined'}
} icon={ @@ -174,7 +172,7 @@ storiesOf('app/ServiceMap/Cytoscape', module) width={80} /> } - title={node.data('label')} + title={node.data('id')} /> ))} diff --git a/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/Cytoscape.tsx b/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/Cytoscape.tsx index 7bdc6aebbd9a0..21bedc204f48b 100644 --- a/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/Cytoscape.tsx +++ b/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/Cytoscape.tsx @@ -124,6 +124,12 @@ export function Cytoscape({ // Trigger a custom "data" event when data changes useEffect(() => { if (cy && elements.length > 0) { + const renderedElements = cy.elements('node,edge'); + const latestElementIds = elements.map(el => el.data.id); + const absentElements = renderedElements.filter( + el => !latestElementIds.includes(el.id()) + ); + cy.remove(absentElements); cy.add(elements); cy.trigger('data'); } diff --git a/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/cytoscapeOptions.ts b/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/cytoscapeOptions.ts index 0438842f7af10..92f66f698f044 100644 --- a/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/cytoscapeOptions.ts +++ b/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/cytoscapeOptions.ts @@ -83,7 +83,7 @@ const style: cytoscape.Stylesheet[] = [ style: { 'curve-style': 'taxi', // @ts-ignore - 'taxi-direction': 'rightward', + 'taxi-direction': 'auto', 'line-color': lineColor, 'overlay-opacity': 0, 'target-arrow-color': lineColor, diff --git a/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/icons.ts b/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/icons.ts index 4925ffba310b5..dd9b48d312725 100644 --- a/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/icons.ts +++ b/x-pack/legacy/plugins/apm/public/components/app/ServiceMap/icons.ts @@ -5,11 +5,12 @@ */ import cytoscape from 'cytoscape'; +import { isRumAgentName } from '../../../../../../../plugins/apm/common/agent_name'; import { AGENT_NAME, SERVICE_NAME, - SPAN_TYPE, - SPAN_SUBTYPE + SPAN_SUBTYPE, + SPAN_TYPE } from '../../../../../../../plugins/apm/common/elasticsearch_fieldnames'; import databaseIcon from './icons/database.svg'; import defaultIconImport from './icons/default.svg'; @@ -62,7 +63,12 @@ export function iconForNode(node: cytoscape.NodeSingular) { const type = node.data(SPAN_TYPE); if (node.data(SERVICE_NAME)) { - return serviceIcons[node.data(AGENT_NAME) as string]; + const agentName = node.data(AGENT_NAME); + // RUM can have multiple names. Normalize it + const normalizedAgentName = isRumAgentName(agentName) + ? 'js-base' + : agentName; + return serviceIcons[normalizedAgentName]; } else if (isIE11) { return defaultIcon; } else if ( diff --git a/x-pack/legacy/plugins/apm/public/components/app/Settings/AgentConfigurations/AgentConfigurationCreateEdit/SettingsPage/SettingFormRow.tsx b/x-pack/legacy/plugins/apm/public/components/app/Settings/AgentConfigurations/AgentConfigurationCreateEdit/SettingsPage/SettingFormRow.tsx index b1959e4d68aa4..30c772bf5f634 100644 --- a/x-pack/legacy/plugins/apm/public/components/app/Settings/AgentConfigurations/AgentConfigurationCreateEdit/SettingsPage/SettingFormRow.tsx +++ b/x-pack/legacy/plugins/apm/public/components/app/Settings/AgentConfigurations/AgentConfigurationCreateEdit/SettingsPage/SettingFormRow.tsx @@ -90,6 +90,7 @@ function FormRow({ onChange( setting.key, diff --git a/x-pack/legacy/plugins/apm/public/components/app/Settings/CustomizeUI/CustomLink/Title.tsx b/x-pack/legacy/plugins/apm/public/components/app/Settings/CustomizeUI/CustomLink/Title.tsx index 17ec42b3e2016..07af7b0c0e7db 100644 --- a/x-pack/legacy/plugins/apm/public/components/app/Settings/CustomizeUI/CustomLink/Title.tsx +++ b/x-pack/legacy/plugins/apm/public/components/app/Settings/CustomizeUI/CustomLink/Title.tsx @@ -28,7 +28,7 @@ export const Title = () => ( 'xpack.apm.settings.customizeUI.customLink.info', { defaultMessage: - "These links will be shown in the 'Actions' context menu for the transaction detail." + 'These links will be shown in the Actions context menu for transactions.' } )} /> diff --git a/x-pack/legacy/plugins/apm/public/components/app/Settings/CustomizeUI/index.tsx b/x-pack/legacy/plugins/apm/public/components/app/Settings/CustomizeUI/index.tsx index 1cd1298fdd549..350f2185fb3c8 100644 --- a/x-pack/legacy/plugins/apm/public/components/app/Settings/CustomizeUI/index.tsx +++ b/x-pack/legacy/plugins/apm/public/components/app/Settings/CustomizeUI/index.tsx @@ -14,8 +14,8 @@ export const CustomizeUI = () => { <>

- {i18n.translate('xpack.apm.settings.customizeUI', { - defaultMessage: 'Customize UI' + {i18n.translate('xpack.apm.settings.customizeApp', { + defaultMessage: 'Customize app' })}

diff --git a/x-pack/legacy/plugins/apm/public/components/app/Settings/index.tsx b/x-pack/legacy/plugins/apm/public/components/app/Settings/index.tsx index f33bb17decd4e..2bb85876686bf 100644 --- a/x-pack/legacy/plugins/apm/public/components/app/Settings/index.tsx +++ b/x-pack/legacy/plugins/apm/public/components/app/Settings/index.tsx @@ -57,8 +57,8 @@ export const Settings: React.FC = props => { isSelected: pathname === '/settings/apm-indices' }, { - name: i18n.translate('xpack.apm.settings.customizeUI', { - defaultMessage: 'Customize UI' + name: i18n.translate('xpack.apm.settings.customizeApp', { + defaultMessage: 'Customize app' }), id: '3', href: getAPMHref('/settings/customize-ui', search), diff --git a/x-pack/legacy/plugins/apm/public/components/shared/ErrorRateAlertTrigger/index.tsx b/x-pack/legacy/plugins/apm/public/components/shared/ErrorRateAlertTrigger/index.tsx index 9bfc5936a555e..b7e23c2979cb8 100644 --- a/x-pack/legacy/plugins/apm/public/components/shared/ErrorRateAlertTrigger/index.tsx +++ b/x-pack/legacy/plugins/apm/public/components/shared/ErrorRateAlertTrigger/index.tsx @@ -6,6 +6,7 @@ import React from 'react'; import { EuiFieldNumber } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; +import { isFinite } from 'lodash'; import { ForLastExpression } from '../../../../../../../plugins/triggers_actions_ui/public'; import { ALERT_TYPES_CONFIG } from '../../../../../../../plugins/apm/common/alert_types'; import { ServiceAlertTrigger } from '../ServiceAlertTrigger'; @@ -37,15 +38,17 @@ export function ErrorRateAlertTrigger(props: Props) { ...alertParams }; + const threshold = isFinite(params.threshold) ? params.threshold : ''; + const fields = [ setAlertParams('threshold', parseInt(e.target.value, 10)) diff --git a/x-pack/legacy/plugins/apm/public/components/shared/TransactionActionMenu/CustomLink/CustomLinkSection.tsx b/x-pack/legacy/plugins/apm/public/components/shared/TransactionActionMenu/CustomLink/CustomLinkSection.tsx index 52befe37ffdae..bd00bcf600ffe 100644 --- a/x-pack/legacy/plugins/apm/public/components/shared/TransactionActionMenu/CustomLink/CustomLinkSection.tsx +++ b/x-pack/legacy/plugins/apm/public/components/shared/TransactionActionMenu/CustomLink/CustomLinkSection.tsx @@ -3,14 +3,26 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import React from 'react'; +import { EuiLink, EuiText } from '@elastic/eui'; import Mustache from 'mustache'; +import React from 'react'; +import styled from 'styled-components'; import { CustomLink } from '../../../../../../../../plugins/apm/common/custom_link/custom_link_types'; import { Transaction } from '../../../../../../../../plugins/apm/typings/es_schemas/ui/transaction'; -import { - SectionLinks, - SectionLink -} from '../../../../../../../../plugins/observability/public'; +import { px, truncate, units } from '../../../../style/variables'; + +const LinkContainer = styled.li` + margin-top: ${px(units.half)}; + &:first-of-type { + margin-top: 0; + } +`; + +const TruncateText = styled(EuiText)` + font-weight: 500; + line-height: ${px(units.unit)}; + ${truncate(px(units.unit * 25))} +`; export const CustomLinkSection = ({ customLinks, @@ -19,7 +31,7 @@ export const CustomLinkSection = ({ customLinks: CustomLink[]; transaction: Transaction; }) => ( - +
    {customLinks.map(link => { let href = link.url; try { @@ -28,13 +40,12 @@ export const CustomLinkSection = ({ // ignores any error that happens } return ( - + + + {link.label} + + ); })} - +
); diff --git a/x-pack/legacy/plugins/apm/public/components/shared/TransactionActionMenu/CustomLink/index.test.tsx b/x-pack/legacy/plugins/apm/public/components/shared/TransactionActionMenu/CustomLink/index.test.tsx index 2dab8d63f99b2..9d1eeb9a3136d 100644 --- a/x-pack/legacy/plugins/apm/public/components/shared/TransactionActionMenu/CustomLink/index.test.tsx +++ b/x-pack/legacy/plugins/apm/public/components/shared/TransactionActionMenu/CustomLink/index.test.tsx @@ -28,7 +28,7 @@ describe('Custom links', () => { ); expectTextsInDocument(component, [ - 'No custom links found. Set up your own custom links i.e. a link to a specific Dashboard or external link.' + 'No custom links found. Set up your own custom links, e.g., a link to a specific Dashboard or external link.' ]); expectTextsNotInDocument(component, ['Create']); }); diff --git a/x-pack/legacy/plugins/apm/public/components/shared/TransactionActionMenu/CustomLink/index.tsx b/x-pack/legacy/plugins/apm/public/components/shared/TransactionActionMenu/CustomLink/index.tsx index b32d8f0d9582c..38b672a181fce 100644 --- a/x-pack/legacy/plugins/apm/public/components/shared/TransactionActionMenu/CustomLink/index.tsx +++ b/x-pack/legacy/plugins/apm/public/components/shared/TransactionActionMenu/CustomLink/index.tsx @@ -55,7 +55,7 @@ export const CustomLink = ({ {i18n.translate('xpack.apm.customLink.empty', { defaultMessage: - 'No custom links found. Set up your own custom links i.e. a link to a specific Dashboard or external link.' + 'No custom links found. Set up your own custom links, e.g., a link to a specific Dashboard or external link.' })} diff --git a/x-pack/legacy/plugins/apm/public/components/shared/TransactionActionMenu/TransactionActionMenu.tsx b/x-pack/legacy/plugins/apm/public/components/shared/TransactionActionMenu/TransactionActionMenu.tsx index 048ed662ec502..e3fbcf8485d54 100644 --- a/x-pack/legacy/plugins/apm/public/components/shared/TransactionActionMenu/TransactionActionMenu.tsx +++ b/x-pack/legacy/plugins/apm/public/components/shared/TransactionActionMenu/TransactionActionMenu.tsx @@ -85,7 +85,13 @@ export const TransactionActionMenu: FunctionComponent = ({ urlParams }); + const closePopover = () => { + setIsActionPopoverOpen(false); + setIsCustomLinksPopoverOpen(false); + }; + const toggleCustomLinkFlyout = () => { + closePopover(); setIsCustomLinkFlyoutOpen(isOpen => !isOpen); }; @@ -111,10 +117,7 @@ export const TransactionActionMenu: FunctionComponent = ({ )} { - setIsActionPopoverOpen(false); - setIsCustomLinksPopoverOpen(false); - }} + closePopover={closePopover} isOpen={isActionPopoverOpen} anchorPosition="downRight" button={ diff --git a/x-pack/legacy/plugins/canvas/canvas_plugin_src/templates/index.ts b/x-pack/legacy/plugins/canvas/canvas_plugin_src/templates/index.ts index eba0b47f7dc13..88d2b904e6cb3 100644 --- a/x-pack/legacy/plugins/canvas/canvas_plugin_src/templates/index.ts +++ b/x-pack/legacy/plugins/canvas/canvas_plugin_src/templates/index.ts @@ -7,7 +7,7 @@ import { applyTemplateStrings } from '../../i18n/templates'; import darkTemplate from './theme_dark.json'; import lightTemplate from './theme_light.json'; -import pitchTemplate from './pitch_presentation.json'; +// import pitchTemplate from './pitch_presentation.json'; import statusTemplate from './status_report.json'; import summaryTemplate from './summary_report.json'; @@ -15,7 +15,7 @@ import summaryTemplate from './summary_report.json'; export const templateSpecs = applyTemplateStrings([ darkTemplate, lightTemplate, - pitchTemplate, + // pitchTemplate, statusTemplate, summaryTemplate, ]); diff --git a/x-pack/legacy/plugins/canvas/i18n/templates/template_strings.ts b/x-pack/legacy/plugins/canvas/i18n/templates/template_strings.ts index 5ab6a908641de..d8e4d51706be9 100644 --- a/x-pack/legacy/plugins/canvas/i18n/templates/template_strings.ts +++ b/x-pack/legacy/plugins/canvas/i18n/templates/template_strings.ts @@ -37,14 +37,6 @@ export const getTemplateStrings = (): TemplateStringDict => ({ defaultMessage: 'Light color themed presentation deck', }), }, - Pitch: { - name: i18n.translate('xpack.canvas.templates.pitchName', { - defaultMessage: 'Pitch', - }), - help: i18n.translate('xpack.canvas.templates.pitchHelp', { - defaultMessage: 'Branded presentation with large photos', - }), - }, Status: { name: i18n.translate('xpack.canvas.templates.statusName', { defaultMessage: 'Status', @@ -62,3 +54,14 @@ export const getTemplateStrings = (): TemplateStringDict => ({ }), }, }); + +export const getUnusedTemplateStrings = (): TemplateStringDict => ({ + Pitch: { + name: i18n.translate('xpack.canvas.templates.pitchName', { + defaultMessage: 'Pitch', + }), + help: i18n.translate('xpack.canvas.templates.pitchHelp', { + defaultMessage: 'Branded presentation with large photos', + }), + }, +}); diff --git a/x-pack/legacy/plugins/lens/public/editor_frame_service/service.test.tsx b/x-pack/legacy/plugins/lens/public/editor_frame_service/service.test.tsx index 47fd810bb4c53..42a1fcc055a1e 100644 --- a/x-pack/legacy/plugins/lens/public/editor_frame_service/service.test.tsx +++ b/x-pack/legacy/plugins/lens/public/editor_frame_service/service.test.tsx @@ -17,7 +17,6 @@ import { CoreSetup } from 'kibana/public'; jest.mock('ui/new_platform'); // mock away actual dependencies to prevent all of it being loaded -jest.mock('../../../../../../src/legacy/core_plugins/interpreter/public/registries', () => {}); jest.mock('./embeddable/embeddable_factory', () => ({ EmbeddableFactory: class Mock {}, })); diff --git a/x-pack/legacy/plugins/lens/public/helpers/url_helper.test.ts b/x-pack/legacy/plugins/lens/public/helpers/url_helper.test.ts index 9c59c9a96d00f..ef960fb52952b 100644 --- a/x-pack/legacy/plugins/lens/public/helpers/url_helper.test.ts +++ b/x-pack/legacy/plugins/lens/public/helpers/url_helper.test.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -jest.mock('../legacy_imports', () => ({ +jest.mock('../../../../../../src/plugins/dashboard/public', () => ({ DashboardConstants: { ADD_EMBEDDABLE_ID: 'addEmbeddableId', ADD_EMBEDDABLE_TYPE: 'addEmbeddableType', diff --git a/x-pack/legacy/plugins/lens/public/helpers/url_helper.ts b/x-pack/legacy/plugins/lens/public/helpers/url_helper.ts index fca44195b98c4..3495c15118ce7 100644 --- a/x-pack/legacy/plugins/lens/public/helpers/url_helper.ts +++ b/x-pack/legacy/plugins/lens/public/helpers/url_helper.ts @@ -5,7 +5,7 @@ */ import { parseUrl, stringify } from 'query-string'; -import { DashboardConstants } from '../legacy_imports'; +import { DashboardConstants } from '../../../../../../src/plugins/dashboard/public'; type UrlVars = Record; diff --git a/x-pack/legacy/plugins/lens/public/legacy.ts b/x-pack/legacy/plugins/lens/public/legacy.ts index b7d47644c7f31..3b7b6a7a1b510 100644 --- a/x-pack/legacy/plugins/lens/public/legacy.ts +++ b/x-pack/legacy/plugins/lens/public/legacy.ts @@ -5,7 +5,6 @@ */ import { npSetup, npStart } from 'ui/new_platform'; -import { visualizations } from './legacy_imports'; export * from './types'; @@ -14,6 +13,5 @@ import { plugin } from './index'; const pluginInstance = plugin(); pluginInstance.setup(npSetup.core, { ...npSetup.plugins, - __LEGACY: { visualizations }, }); pluginInstance.start(npStart.core, npStart.plugins); diff --git a/x-pack/legacy/plugins/lens/public/legacy_imports.ts b/x-pack/legacy/plugins/lens/public/legacy_imports.ts deleted file mode 100644 index 0f37e460e2957..0000000000000 --- a/x-pack/legacy/plugins/lens/public/legacy_imports.ts +++ /dev/null @@ -1,10 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { npSetup } from 'ui/new_platform'; -export const { visualizations } = npSetup.plugins; -export { VisualizationsSetup } from '../../../../../src/plugins/visualizations/public'; -export { DashboardConstants } from '../../../../../src/plugins/dashboard/public'; diff --git a/x-pack/legacy/plugins/lens/public/plugin.tsx b/x-pack/legacy/plugins/lens/public/plugin.tsx index 45817fdc3c05f..b426a12d07f9b 100644 --- a/x-pack/legacy/plugins/lens/public/plugin.tsx +++ b/x-pack/legacy/plugins/lens/public/plugin.tsx @@ -15,7 +15,9 @@ import { AppMountParameters, CoreSetup, CoreStart } from 'src/core/public'; import { DataPublicPluginSetup, DataPublicPluginStart } from 'src/plugins/data/public'; import { EmbeddableSetup, EmbeddableStart } from 'src/plugins/embeddable/public'; import { ExpressionsSetup, ExpressionsStart } from 'src/plugins/expressions/public'; +import { VisualizationsSetup } from 'src/plugins/visualizations/public'; import { KibanaLegacySetup } from 'src/plugins/kibana_legacy/public'; +import { DashboardConstants } from '../../../../../src/plugins/dashboard/public'; import { Storage } from '../../../../../src/plugins/kibana_utils/public'; import { EditorFrameService } from './editor_frame_service'; import { IndexPatternDatasource } from './indexpattern_datasource'; @@ -37,16 +39,13 @@ import { NOT_INTERNATIONALIZED_PRODUCT_NAME } from '../../../../plugins/lens/com import { addEmbeddableToDashboardUrl, getUrlVars } from './helpers'; import { EditorFrameStart } from './types'; import { getLensAliasConfig } from './vis_type_alias'; -import { VisualizationsSetup, DashboardConstants } from './legacy_imports'; export interface LensPluginSetupDependencies { kibanaLegacy: KibanaLegacySetup; expressions: ExpressionsSetup; data: DataPublicPluginSetup; embeddable: EmbeddableSetup; - __LEGACY: { - visualizations: VisualizationsSetup; - }; + visualizations: VisualizationsSetup; } export interface LensPluginStartDependencies { @@ -77,13 +76,7 @@ export class LensPlugin { setup( core: CoreSetup, - { - kibanaLegacy, - expressions, - data, - embeddable, - __LEGACY: { visualizations }, - }: LensPluginSetupDependencies + { kibanaLegacy, expressions, data, embeddable, visualizations }: LensPluginSetupDependencies ) { const editorFrameSetupInterface = this.editorFrameService.setup(core, { data, diff --git a/x-pack/legacy/plugins/monitoring/common/constants.ts b/x-pack/legacy/plugins/monitoring/common/constants.ts index 9a4030f3eb214..3a4c7b71dcd03 100644 --- a/x-pack/legacy/plugins/monitoring/common/constants.ts +++ b/x-pack/legacy/plugins/monitoring/common/constants.ts @@ -239,11 +239,15 @@ export const ALERT_TYPE_PREFIX = 'monitoring_'; * This is the alert type id for the license expiration alert */ export const ALERT_TYPE_LICENSE_EXPIRATION = `${ALERT_TYPE_PREFIX}alert_type_license_expiration`; +/** + * This is the alert type id for the cluster state alert + */ +export const ALERT_TYPE_CLUSTER_STATE = `${ALERT_TYPE_PREFIX}alert_type_cluster_state`; /** * A listing of all alert types */ -export const ALERT_TYPES = [ALERT_TYPE_LICENSE_EXPIRATION]; +export const ALERT_TYPES = [ALERT_TYPE_LICENSE_EXPIRATION, ALERT_TYPE_CLUSTER_STATE]; /** * Matches the id for the built-in in email action type @@ -254,7 +258,7 @@ export const ALERT_ACTION_TYPE_EMAIL = '.email'; /** * The number of alerts that have been migrated */ -export const NUMBER_OF_MIGRATED_ALERTS = 1; +export const NUMBER_OF_MIGRATED_ALERTS = 2; /** * The advanced settings config name for the email address diff --git a/x-pack/legacy/plugins/monitoring/public/components/alerts/alerts.js b/x-pack/legacy/plugins/monitoring/public/components/alerts/alerts.js index 11fcef73a4b97..95c1af5549198 100644 --- a/x-pack/legacy/plugins/monitoring/public/components/alerts/alerts.js +++ b/x-pack/legacy/plugins/monitoring/public/components/alerts/alerts.js @@ -6,10 +6,15 @@ import React from 'react'; import chrome from '../../np_imports/ui/chrome'; -import { capitalize } from 'lodash'; +import { capitalize, get } from 'lodash'; import { formatDateTimeLocal } from '../../../common/formatting'; import { formatTimestampToDuration } from '../../../common'; -import { CALCULATE_DURATION_SINCE, EUI_SORT_DESCENDING } from '../../../common/constants'; +import { + CALCULATE_DURATION_SINCE, + EUI_SORT_DESCENDING, + ALERT_TYPE_LICENSE_EXPIRATION, + ALERT_TYPE_CLUSTER_STATE, +} from '../../../common/constants'; import { mapSeverity } from './map_severity'; import { FormattedAlert } from 'plugins/monitoring/components/alerts/formatted_alert'; import { EuiMonitoringTable } from 'plugins/monitoring/components/table'; @@ -21,6 +26,8 @@ const linkToCategories = { 'elasticsearch/indices': 'Elasticsearch Indices', 'kibana/instances': 'Kibana Instances', 'logstash/instances': 'Logstash Nodes', + [ALERT_TYPE_LICENSE_EXPIRATION]: 'License expiration', + [ALERT_TYPE_CLUSTER_STATE]: 'Cluster state', }; const getColumns = (kbnUrl, scope, timezone) => [ { @@ -94,19 +101,22 @@ const getColumns = (kbnUrl, scope, timezone) => [ }), field: 'message', sortable: true, - render: (message, alert) => ( - { - scope.$evalAsync(() => { - kbnUrl.changePath(target); - }); - }} - /> - ), + render: (_message, alert) => { + const message = get(alert, 'message.text', get(alert, 'message', '')); + return ( + { + scope.$evalAsync(() => { + kbnUrl.changePath(target); + }); + }} + /> + ); + }, }, { name: i18n.translate('xpack.monitoring.alerts.categoryColumnTitle', { @@ -148,8 +158,8 @@ const getColumns = (kbnUrl, scope, timezone) => [ export const Alerts = ({ alerts, angular, sorting, pagination, onTableChange }) => { const alertsFlattened = alerts.map(alert => ({ ...alert, - status: alert.metadata.severity, - category: alert.metadata.link, + status: get(alert, 'metadata.severity', get(alert, 'severity', 0)), + category: get(alert, 'metadata.link', get(alert, 'type', null)), })); const injector = chrome.dangerouslyGetActiveInjector(); diff --git a/x-pack/legacy/plugins/monitoring/public/components/alerts/status.test.tsx b/x-pack/legacy/plugins/monitoring/public/components/alerts/status.test.tsx index 258a5b68db372..d3cf4b463a2cc 100644 --- a/x-pack/legacy/plugins/monitoring/public/components/alerts/status.test.tsx +++ b/x-pack/legacy/plugins/monitoring/public/components/alerts/status.test.tsx @@ -8,7 +8,7 @@ import React from 'react'; import { shallow } from 'enzyme'; import { kfetch } from 'ui/kfetch'; import { AlertsStatus, AlertsStatusProps } from './status'; -import { ALERT_TYPE_PREFIX } from '../../../common/constants'; +import { ALERT_TYPES } from '../../../common/constants'; import { getSetupModeState } from '../../lib/setup_mode'; import { mockUseEffects } from '../../jest.helpers'; @@ -63,11 +63,7 @@ describe('Status', () => { it('should render a success message if all alerts have been migrated and in setup mode', async () => { (kfetch as jest.Mock).mockReturnValue({ - data: [ - { - alertTypeId: ALERT_TYPE_PREFIX, - }, - ], + data: ALERT_TYPES.map(type => ({ alertTypeId: type })), }); (getSetupModeState as jest.Mock).mockReturnValue({ diff --git a/x-pack/legacy/plugins/monitoring/public/components/alerts/status.tsx b/x-pack/legacy/plugins/monitoring/public/components/alerts/status.tsx index 072a98b123452..5f5329bf7fff8 100644 --- a/x-pack/legacy/plugins/monitoring/public/components/alerts/status.tsx +++ b/x-pack/legacy/plugins/monitoring/public/components/alerts/status.tsx @@ -142,7 +142,7 @@ export const AlertsStatus: React.FC = (props: AlertsStatusPro ); } - const allMigrated = kibanaAlerts.length === NUMBER_OF_MIGRATED_ALERTS; + const allMigrated = kibanaAlerts.length >= NUMBER_OF_MIGRATED_ALERTS; if (allMigrated) { if (setupModeEnabled) { return ( diff --git a/x-pack/legacy/plugins/monitoring/public/components/cluster/overview/alerts_panel.js b/x-pack/legacy/plugins/monitoring/public/components/cluster/overview/alerts_panel.js index 8455fb8cf3088..d87ff98e79be0 100644 --- a/x-pack/legacy/plugins/monitoring/public/components/cluster/overview/alerts_panel.js +++ b/x-pack/legacy/plugins/monitoring/public/components/cluster/overview/alerts_panel.js @@ -6,14 +6,12 @@ import React, { Fragment } from 'react'; import moment from 'moment-timezone'; -import chrome from '../../../np_imports/ui/chrome'; import { FormattedAlert } from 'plugins/monitoring/components/alerts/formatted_alert'; import { mapSeverity } from 'plugins/monitoring/components/alerts/map_severity'; import { formatTimestampToDuration } from '../../../../common/format_timestamp_to_duration'; import { CALCULATE_DURATION_SINCE, KIBANA_ALERTING_ENABLED, - ALERT_TYPE_LICENSE_EXPIRATION, CALCULATE_DURATION_UNTIL, } from '../../../../common/constants'; import { formatDateTimeLocal } from '../../../../common/formatting'; @@ -31,6 +29,37 @@ import { EuiLink, } from '@elastic/eui'; +function replaceTokens(alert) { + if (!alert.message.tokens) { + return alert.message.text; + } + + let text = alert.message.text; + + for (const token of alert.message.tokens) { + if (token.type === 'time') { + text = text.replace( + token.startToken, + token.isRelative + ? formatTimestampToDuration(alert.expirationTime, CALCULATE_DURATION_UNTIL) + : moment.tz(alert.expirationTime, moment.tz.guess()).format('LLL z') + ); + } else if (token.type === 'link') { + const linkPart = new RegExp(`${token.startToken}(.+?)${token.endToken}`).exec(text); + // TODO: we assume this is at the end, which works for now but will not always work + const nonLinkText = text.replace(linkPart[0], ''); + text = ( + + {nonLinkText} + {linkPart[1]} + + ); + } + } + + return text; +} + export function AlertsPanel({ alerts, changeUrl }) { const goToAlerts = () => changeUrl('/alerts'); @@ -58,9 +87,6 @@ export function AlertsPanel({ alerts, changeUrl }) { severityIcon.iconType = 'check'; } - const injector = chrome.dangerouslyGetActiveInjector(); - const timezone = injector.get('config').get('dateFormat:tz'); - return ( @@ -96,14 +122,7 @@ export function AlertsPanel({ alerts, changeUrl }) { const alertsList = KIBANA_ALERTING_ENABLED ? alerts.map((alert, idx) => { const callOutProps = mapSeverity(alert.severity); - let message = alert.message - // scan message prefix and replace relative times - // \w: Matches any alphanumeric character from the basic Latin alphabet, including the underscore. Equivalent to [A-Za-z0-9_]. - .replace( - '#relative', - formatTimestampToDuration(alert.expirationTime, CALCULATE_DURATION_UNTIL) - ) - .replace('#absolute', moment.tz(alert.expirationTime, moment.tz.guess()).format('LLL z')); + const message = replaceTokens(alert); if (!alert.isFiring) { callOutProps.title = i18n.translate( @@ -118,22 +137,30 @@ export function AlertsPanel({ alerts, changeUrl }) { ); callOutProps.color = 'success'; callOutProps.iconType = 'check'; - } else { - if (alert.type === ALERT_TYPE_LICENSE_EXPIRATION) { - message = ( - - {message} -   - Please update your license - - ); - } } return ( - -

{message}

- + + +

{message}

+ +

+ +

+ + + + ); }) : alerts.map((item, index) => ( diff --git a/x-pack/legacy/plugins/monitoring/public/views/alerts/index.js b/x-pack/legacy/plugins/monitoring/public/views/alerts/index.js index 7c065a78a8af9..62cc985887e9f 100644 --- a/x-pack/legacy/plugins/monitoring/public/views/alerts/index.js +++ b/x-pack/legacy/plugins/monitoring/public/views/alerts/index.js @@ -18,25 +18,37 @@ import { Alerts } from '../../components/alerts'; import { MonitoringViewBaseEuiTableController } from '../base_eui_table_controller'; import { FormattedMessage } from '@kbn/i18n/react'; import { EuiPage, EuiPageBody, EuiPageContent, EuiSpacer, EuiLink } from '@elastic/eui'; -import { CODE_PATH_ALERTS } from '../../../common/constants'; +import { CODE_PATH_ALERTS, KIBANA_ALERTING_ENABLED } from '../../../common/constants'; function getPageData($injector) { const globalState = $injector.get('globalState'); const $http = $injector.get('$http'); const Private = $injector.get('Private'); - const url = `../api/monitoring/v1/clusters/${globalState.cluster_uuid}/legacy_alerts`; + const url = KIBANA_ALERTING_ENABLED + ? `../api/monitoring/v1/alert_status` + : `../api/monitoring/v1/clusters/${globalState.cluster_uuid}/legacy_alerts`; const timeBounds = timefilter.getBounds(); + const data = { + timeRange: { + min: timeBounds.min.toISOString(), + max: timeBounds.max.toISOString(), + }, + }; + + if (!KIBANA_ALERTING_ENABLED) { + data.ccs = globalState.ccs; + } return $http - .post(url, { - ccs: globalState.ccs, - timeRange: { - min: timeBounds.min.toISOString(), - max: timeBounds.max.toISOString(), - }, + .post(url, data) + .then(response => { + const result = get(response, 'data', []); + if (KIBANA_ALERTING_ENABLED) { + return result.alerts; + } + return result; }) - .then(response => get(response, 'data', [])) .catch(err => { const ajaxErrorHandlers = Private(ajaxErrorHandlersProvider); return ajaxErrorHandlers(err); diff --git a/x-pack/legacy/plugins/siem/cypress/integration/signal_detection_rules.spec.ts b/x-pack/legacy/plugins/siem/cypress/integration/signal_detection_rules.spec.ts new file mode 100644 index 0000000000000..1559285d508ed --- /dev/null +++ b/x-pack/legacy/plugins/siem/cypress/integration/signal_detection_rules.spec.ts @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { + FIFTH_RULE, + FIRST_RULE, + RULE_NAME, + SECOND_RULE, + SEVENTH_RULE, +} from '../screens/signal_detection_rules'; + +import { goToManageSignalDetectionRules } from '../tasks/detections'; +import { esArchiverLoad, esArchiverUnload } from '../tasks/es_archiver'; +import { loginAndWaitForPageWithoutDateRange } from '../tasks/login'; +import { + activateRule, + sortByActivatedRules, + waitForLoadElasticPrebuiltDetectionRulesTableToBeLoaded, + waitForRuleToBeActivated, +} from '../tasks/signal_detection_rules'; + +import { DETECTIONS } from '../urls/navigation'; + +describe('Signal detection rules', () => { + before(() => { + esArchiverLoad('prebuilt_rules_loaded'); + }); + + after(() => { + esArchiverUnload('prebuilt_rules_loaded'); + }); + + it('Sorts by activated rules', () => { + loginAndWaitForPageWithoutDateRange(DETECTIONS); + goToManageSignalDetectionRules(); + waitForLoadElasticPrebuiltDetectionRulesTableToBeLoaded(); + cy.get(RULE_NAME) + .eq(FIFTH_RULE) + .invoke('text') + .then(fifthRuleName => { + activateRule(FIFTH_RULE); + waitForRuleToBeActivated(); + cy.get(RULE_NAME) + .eq(SEVENTH_RULE) + .invoke('text') + .then(seventhRuleName => { + activateRule(SEVENTH_RULE); + waitForRuleToBeActivated(); + sortByActivatedRules(); + + cy.get(RULE_NAME) + .eq(FIRST_RULE) + .should('have.text', fifthRuleName); + cy.get(RULE_NAME) + .eq(SECOND_RULE) + .should('have.text', seventhRuleName); + }); + }); + }); +}); diff --git a/x-pack/legacy/plugins/siem/cypress/screens/signal_detection_rules.ts b/x-pack/legacy/plugins/siem/cypress/screens/signal_detection_rules.ts index 09fbc2132302c..f74f5c26ddc2e 100644 --- a/x-pack/legacy/plugins/siem/cypress/screens/signal_detection_rules.ts +++ b/x-pack/legacy/plugins/siem/cypress/screens/signal_detection_rules.ts @@ -18,6 +18,10 @@ export const DELETE_RULE_BULK_BTN = '[data-test-subj="deleteRuleBulk"]'; export const ELASTIC_RULES_BTN = '[data-test-subj="show-elastic-rules-filter-button"]'; +export const FIFTH_RULE = 4; + +export const FIRST_RULE = 0; + export const LOAD_PREBUILT_RULES_BTN = '[data-test-subj="load-prebuilt-rules"]'; export const LOADING_INITIAL_PREBUILT_RULES_TABLE = @@ -31,18 +35,26 @@ export const RISK_SCORE = '[data-test-subj="riskScore"]'; export const RELOAD_PREBUILT_RULES_BTN = '[data-test-subj="reloadPrebuiltRulesBtn"]'; +export const SECOND_RULE = 1; + export const RULE_CHECKBOX = '.euiTableRow .euiCheckbox__input'; export const RULE_NAME = '[data-test-subj="ruleName"]'; export const RULE_SWITCH = '[data-test-subj="rule-switch"]'; +export const RULE_SWITCH_LOADER = '[data-test-subj="rule-switch-loader"]'; + export const RULES_TABLE = '[data-test-subj="rules-table"]'; export const RULES_ROW = '.euiTableRow'; +export const SEVENTH_RULE = 6; + export const SEVERITY = '[data-test-subj="severity"]'; export const SHOWING_RULES_TEXT = '[data-test-subj="showingRules"]'; +export const SORT_RULES_BTN = '[data-test-subj="tableHeaderSortButton"]'; + export const THREE_HUNDRED_ROWS = '[data-test-subj="tablePagination-300-rows"]'; diff --git a/x-pack/legacy/plugins/siem/cypress/tasks/signal_detection_rules.ts b/x-pack/legacy/plugins/siem/cypress/tasks/signal_detection_rules.ts index cfc490526d84e..a404f1142cba7 100644 --- a/x-pack/legacy/plugins/siem/cypress/tasks/signal_detection_rules.ts +++ b/x-pack/legacy/plugins/siem/cypress/tasks/signal_detection_rules.ts @@ -15,13 +15,22 @@ import { LOADING_INITIAL_PREBUILT_RULES_TABLE, LOADING_SPINNER, PAGINATION_POPOVER_BTN, + RELOAD_PREBUILT_RULES_BTN, RULE_CHECKBOX, RULE_NAME, + RULE_SWITCH, + RULE_SWITCH_LOADER, RULES_TABLE, + SORT_RULES_BTN, THREE_HUNDRED_ROWS, - RELOAD_PREBUILT_RULES_BTN, } from '../screens/signal_detection_rules'; +export const activateRule = (rulePosition: number) => { + cy.get(RULE_SWITCH) + .eq(rulePosition) + .click({ force: true }); +}; + export const changeToThreeHundredRowsPerPage = () => { cy.get(PAGINATION_POPOVER_BTN).click({ force: true }); cy.get(THREE_HUNDRED_ROWS).click(); @@ -71,6 +80,13 @@ export const selectNumberOfRules = (numberOfRules: number) => { } }; +export const sortByActivatedRules = () => { + cy.get(SORT_RULES_BTN).click({ force: true }); + waitForRulesToBeLoaded(); + cy.get(SORT_RULES_BTN).click({ force: true }); + waitForRulesToBeLoaded(); +}; + export const waitForLoadElasticPrebuiltDetectionRulesTableToBeLoaded = () => { cy.get(LOADING_INITIAL_PREBUILT_RULES_TABLE).should('exist'); cy.get(LOADING_INITIAL_PREBUILT_RULES_TABLE).should('not.exist'); @@ -81,6 +97,11 @@ export const waitForPrebuiltDetectionRulesToBeLoaded = () => { cy.get(RULES_TABLE).should('exist'); }; +export const waitForRuleToBeActivated = () => { + cy.get(RULE_SWITCH_LOADER).should('exist'); + cy.get(RULE_SWITCH_LOADER).should('not.exist'); +}; + export const waitForRulesToBeLoaded = () => { cy.get(LOADING_SPINNER).should('exist'); cy.get(LOADING_SPINNER).should('not.exist'); diff --git a/x-pack/legacy/plugins/siem/server/lib/index_fields/elasticsearch_adapter.test.ts b/x-pack/legacy/plugins/siem/server/lib/index_fields/elasticsearch_adapter.test.ts index 93472b8539efd..20bc1387a3c4e 100644 --- a/x-pack/legacy/plugins/siem/server/lib/index_fields/elasticsearch_adapter.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/index_fields/elasticsearch_adapter.test.ts @@ -23,77 +23,108 @@ describe('Index Fields', () => { ).toEqual( sortBy('name', [ { - aggregatable: true, - category: 'base', description: - 'Date/time when the event originated. For log events this is the date/time when the event was generated, and not when it was read. Required field for all events.', + 'Date/time when the event originated.\n\nThis is the date/time extracted from the event, typically representing when\nthe event was generated by the source.\n\nIf the event source has no original timestamp, this value is typically populated\nby the first time the event was received by the pipeline.\n\nRequired field for all events.', example: '2016-05-23T08:05:34.853Z', - indexes: ['auditbeat', 'filebeat', 'packetbeat'], name: '@timestamp', - searchable: true, type: 'date', + searchable: true, + aggregatable: true, + category: 'base', + indexes: ['auditbeat', 'filebeat', 'packetbeat'], }, { + description: 'Each document has an _id that uniquely identifies it', + example: 'Y-6TfmcB0WOhS6qyMv3s', + footnote: '', + group: 1, + level: 'core', + name: '_id', + required: true, + type: 'string', + searchable: true, + aggregatable: false, + readFromDocValues: true, + category: '_id', + indexes: ['auditbeat', 'filebeat', 'packetbeat'], + }, + { + description: + 'An index is like a ‘database’ in a relational database. It has a mapping which defines multiple types. An index is a logical namespace which maps to one or more primary shards and can have zero or more replica shards.', + example: 'auditbeat-8.0.0-2019.02.19-000001', + footnote: '', + group: 1, + level: 'core', + name: '_index', + required: true, + type: 'string', + searchable: true, aggregatable: true, - category: 'agent', + readFromDocValues: true, + category: '_index', + indexes: ['auditbeat', 'filebeat', 'packetbeat'], + }, + { description: - 'Ephemeral identifier of this agent (if one exists). This id normally changes across restarts, but `agent.id` does not.', + 'Ephemeral identifier of this agent (if one exists).\n\nThis id normally changes across restarts, but `agent.id` does not.', example: '8a4f500f', - indexes: ['auditbeat'], name: 'agent.ephemeral_id', - searchable: true, type: 'string', - }, - { + searchable: true, aggregatable: true, category: 'agent', - indexes: ['filebeat'], + indexes: ['auditbeat'], + }, + { name: 'agent.hostname', searchable: true, type: 'string', - }, - { aggregatable: true, category: 'agent', + indexes: ['filebeat'], + }, + { description: - 'Unique identifier of this agent (if one exists). Example: For Beats this would be beat.id.', + 'Unique identifier of this agent (if one exists).\n\nExample: For Beats this would be beat.id.', example: '8a4f500d', - indexes: ['packetbeat'], name: 'agent.id', - searchable: true, type: 'string', - }, - { + searchable: true, aggregatable: true, category: 'agent', + indexes: ['packetbeat'], + }, + { description: - 'Name of the agent. This is a name that can be given to an agent. This can be helpful if for example two Filebeat instances are running on the same host but a human readable separation is needed on which Filebeat instance data is coming from. If no name is given, the name is often left empty.', + 'Custom name of the agent.\n\nThis is a name that can be given to an agent. This can be helpful if for example\ntwo Filebeat instances are running on the same host but a human readable separation\nis needed on which Filebeat instance data is coming from.\n\nIf no name is given, the name is often left empty.', example: 'foo', - indexes: ['auditbeat', 'filebeat'], name: 'agent.name', - searchable: true, type: 'string', - }, - { + searchable: true, aggregatable: true, category: 'agent', + indexes: ['auditbeat', 'filebeat'], + }, + { description: - 'Type of the agent. The agent type stays always the same and should be given by the agent used. In case of Filebeat the agent would always be Filebeat also if two Filebeat instances are run on the same machine.', + 'Type of the agent.\n\nThe agent type stays always the same and should be given by the agent used.\nIn case of Filebeat the agent would always be Filebeat also if two Filebeat\ninstances are run on the same machine.', example: 'filebeat', - indexes: ['auditbeat', 'packetbeat'], name: 'agent.type', - searchable: true, type: 'string', - }, - { + searchable: true, aggregatable: true, category: 'agent', + indexes: ['auditbeat', 'packetbeat'], + }, + { description: 'Version of the agent.', example: '6.0.0-rc2', - indexes: ['auditbeat', 'filebeat'], name: 'agent.version', - searchable: true, type: 'string', + searchable: true, + aggregatable: true, + category: 'agent', + indexes: ['auditbeat', 'filebeat'], }, ]) ); diff --git a/x-pack/legacy/plugins/siem/server/lib/index_fields/elasticsearch_adapter.ts b/x-pack/legacy/plugins/siem/server/lib/index_fields/elasticsearch_adapter.ts index 48b9750ddd949..7cfb5ad391149 100644 --- a/x-pack/legacy/plugins/siem/server/lib/index_fields/elasticsearch_adapter.ts +++ b/x-pack/legacy/plugins/siem/server/lib/index_fields/elasticsearch_adapter.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { get } from 'lodash/fp'; +import { isEmpty, get } from 'lodash/fp'; import { IndexField } from '../../graphql/types'; import { @@ -51,6 +51,23 @@ export class ElasticsearchIndexFieldAdapter implements FieldsAdapter { } } +const missingFields = [ + { + name: '_id', + type: 'string', + searchable: true, + aggregatable: false, + readFromDocValues: true, + }, + { + name: '_index', + type: 'string', + searchable: true, + aggregatable: true, + readFromDocValues: true, + }, +]; + export const formatIndexFields = ( responsesIndexFields: IndexFieldDescriptor[][], indexesAlias: IndexAlias[] @@ -59,20 +76,23 @@ export const formatIndexFields = ( .reduce( (accumulator: IndexField[], indexFields: IndexFieldDescriptor[], indexesAliasIdx: number) => [ ...accumulator, - ...indexFields.reduce((itemAccumulator: IndexField[], index: IndexFieldDescriptor) => { - const alias: IndexAlias = indexesAlias[indexesAliasIdx]; - const splitName = index.name.split('.'); - const category = baseCategoryFields.includes(splitName[0]) ? 'base' : splitName[0]; - return [ - ...itemAccumulator, - { - ...(hasDocumentation(alias, index.name) ? getDocumentation(alias, index.name) : {}), - ...index, - category, - indexes: [alias], - } as IndexField, - ]; - }, []), + ...[...missingFields, ...indexFields].reduce( + (itemAccumulator: IndexField[], index: IndexFieldDescriptor) => { + const alias: IndexAlias = indexesAlias[indexesAliasIdx]; + const splitName = index.name.split('.'); + const category = baseCategoryFields.includes(splitName[0]) ? 'base' : splitName[0]; + return [ + ...itemAccumulator, + { + ...(hasDocumentation(alias, index.name) ? getDocumentation(alias, index.name) : {}), + ...index, + category, + indexes: [alias], + } as IndexField, + ]; + }, + [] + ), ], [] ) @@ -84,7 +104,10 @@ export const formatIndexFields = ( ...accumulator.slice(0, alreadyExistingIndexField), { ...existingIndexField, - indexes: [...existingIndexField.indexes, ...indexfield.indexes], + description: isEmpty(existingIndexField.description) + ? indexfield.description + : existingIndexField.description, + indexes: Array.from(new Set([...existingIndexField.indexes, ...indexfield.indexes])), }, ...accumulator.slice(alreadyExistingIndexField + 1), ]; diff --git a/x-pack/legacy/plugins/siem/server/utils/beat_schema/8.0.0/auditbeat.ts b/x-pack/legacy/plugins/siem/server/utils/beat_schema/8.0.0/auditbeat.ts index 773d4ed6a7e95..76c865679dd05 100644 --- a/x-pack/legacy/plugins/siem/server/utils/beat_schema/8.0.0/auditbeat.ts +++ b/x-pack/legacy/plugins/siem/server/utils/beat_schema/8.0.0/auditbeat.ts @@ -15,91 +15,129 @@ export const auditbeatSchema: Schema = [ { key: 'ecs', title: 'ECS', - description: 'ECS fields.', + description: 'ECS Fields.', fields: [ { name: '@timestamp', - type: 'date', level: 'core', required: true, - example: '2016-05-23T08:05:34.853Z', + type: 'date', description: - 'Date/time when the event originated. For log events this is the date/time when the event was generated, and not when it was read. Required field for all events.', - }, - { - name: 'tags', - level: 'core', - type: 'keyword', - example: '["production", "env2"]', - description: 'List of keywords used to tag each event.', + 'Date/time when the event originated.\n\nThis is the date/time extracted from the event, typically representing when\nthe event was generated by the source.\n\nIf the event source has no original timestamp, this value is typically populated\nby the first time the event was received by the pipeline.\n\nRequired field for all events.', + example: '2016-05-23T08:05:34.853Z', }, { name: 'labels', level: 'core', type: 'object', - example: { - env: 'production', - application: 'foo-bar', - }, + object_type: 'keyword', description: - 'Key/value pairs. Can be used to add meta information to events. Should not contain nested objects. All values are stored as keyword. Example: `docker` and `k8s` labels.', + 'Custom key/value pairs.\n\nCan be used to add meta information to events. Should not contain nested objects.\nAll values are stored as keyword.\n\nExample: `docker` and `k8s` labels.', + example: '{"application": "foo-bar", "env": "production"}', }, { name: 'message', level: 'core', type: 'text', - example: 'Hello World', description: - 'For log events the message field contains the log message. In other use cases the message field can be used to concatenate different values which are then freely searchable. If multiple messages exist, they can be combined into one message.', + 'For log events the message field contains the log message, optimized\nfor viewing in a log viewer.\n\nFor structured logs without an original message field, other fields can be concatenated\nto form a human-readable summary of the event.\n\nIf multiple messages exist, they can be combined into one message.', + example: 'Hello World', + }, + { + name: 'tags', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'List of keywords used to tag each event.', + example: '["production", "env2"]', }, { name: 'agent', title: 'Agent', group: 2, description: - 'The agent fields contain the data about the software entity, if any, that collects, detects, or observes events on a host, or takes measurements on a host. Examples include Beats. Agents may also run on observers. ECS agent.* fields shall be populated with details of the agent running on the host or observer where the event happened or the measurement was taken.', + 'The agent fields contain the data about the software entity, if\nany, that collects, detects, or observes events on a host, or takes measurements\non a host.\n\nExamples include Beats. Agents may also run on observers. ECS agent.* fields\nshall be populated with details of the agent running on the host or observer\nwhere the event happened or the measurement was taken.', footnote: - 'Examples: In the case of Beats for logs, the agent.name is filebeat. For APM, it is the agent running in the app/service. The agent information does not change if data is sent through queuing systems like Kafka, Redis, or processing systems such as Logstash or APM Server.', + 'Examples: In the case of Beats for logs, the agent.name is filebeat.\nFor APM, it is the agent running in the app/service. The agent information does\nnot change if data is sent through queuing systems like Kafka, Redis, or processing\nsystems such as Logstash or APM Server.', type: 'group', fields: [ { - name: 'version', + name: 'ephemeral_id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Ephemeral identifier of this agent (if one exists).\n\nThis id normally changes across restarts, but `agent.id` does not.', + example: '8a4f500f', + }, + { + name: 'id', level: 'core', type: 'keyword', - description: 'Version of the agent.', - example: '6.0.0-rc2', + ignore_above: 1024, + description: + 'Unique identifier of this agent (if one exists).\n\nExample: For Beats this would be beat.id.', + example: '8a4f500d', }, { name: 'name', level: 'core', type: 'keyword', + ignore_above: 1024, description: - 'Name of the agent. This is a name that can be given to an agent. This can be helpful if for example two Filebeat instances are running on the same host but a human readable separation is needed on which Filebeat instance data is coming from. If no name is given, the name is often left empty.', + 'Custom name of the agent.\n\nThis is a name that can be given to an agent. This can be helpful if for example\ntwo Filebeat instances are running on the same host but a human readable separation\nis needed on which Filebeat instance data is coming from.\n\nIf no name is given, the name is often left empty.', example: 'foo', }, { name: 'type', level: 'core', type: 'keyword', + ignore_above: 1024, description: - 'Type of the agent. The agent type stays always the same and should be given by the agent used. In case of Filebeat the agent would always be Filebeat also if two Filebeat instances are run on the same machine.', + 'Type of the agent.\n\nThe agent type stays always the same and should be given by the agent used.\nIn case of Filebeat the agent would always be Filebeat also if two Filebeat\ninstances are run on the same machine.', example: 'filebeat', }, { - name: 'id', + name: 'version', level: 'core', type: 'keyword', + ignore_above: 1024, + description: 'Version of the agent.', + example: '6.0.0-rc2', + }, + ], + }, + { + name: 'as', + title: 'Autonomous System', + group: 2, + description: + 'An autonomous system (AS) is a collection of connected Internet Protocol\n(IP) routing prefixes under the control of one or more network operators on\nbehalf of a single administrative entity or domain that presents a common, clearly\ndefined routing policy to the internet.', + type: 'group', + fields: [ + { + name: 'number', + level: 'extended', + type: 'long', description: - 'Unique identifier of this agent (if one exists). Example: For Beats this would be beat.id.', - example: '8a4f500d', + 'Unique number allocated to the autonomous system. The autonomous\nsystem number (ASN) uniquely identifies each network on the Internet.', + example: 15169, }, { - name: 'ephemeral_id', + name: 'organization.name', level: 'extended', type: 'keyword', - description: - 'Ephemeral identifier of this agent (if one exists). This id normally changes across restarts, but `agent.id` does not.', - example: '8a4f500f', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Organization name.', + example: 'Google LLC', }, ], }, @@ -108,4693 +146,7750 @@ export const auditbeatSchema: Schema = [ title: 'Client', group: 2, description: - 'A client is defined as the initiator of a network connection for events regarding sessions, connections, or bidirectional flow records. For TCP events, the client is the initiator of the TCP connection that sends the SYN packet(s). For other protocols, the client is generally the initiator or requestor in the network transaction. Some systems use the term "originator" to refer the client in TCP connections. The client fields describe details about the system acting as the client in the network event. Client fields are usually populated in conjunction with server fields. Client fields are generally not populated for packet-level events. Client / server representations can add semantic context to an exchange, which is helpful to visualize the data in certain situations. If your context falls in that category, you should still ensure that source and destination are filled appropriately.', + 'A client is defined as the initiator of a network connection for\nevents regarding sessions, connections, or bidirectional flow records.\n\nFor TCP events, the client is the initiator of the TCP connection that sends\nthe SYN packet(s). For other protocols, the client is generally the initiator\nor requestor in the network transaction. Some systems use the term "originator"\nto refer the client in TCP connections. The client fields describe details about\nthe system acting as the client in the network event. Client fields are usually\npopulated in conjunction with server fields. Client fields are generally not\npopulated for packet-level events.\n\nClient / server representations can add semantic context to an exchange, which\nis helpful to visualize the data in certain situations. If your context falls\nin that category, you should still ensure that source and destination are filled\nappropriately.', type: 'group', fields: [ { name: 'address', level: 'extended', type: 'keyword', + ignore_above: 1024, description: - 'Some event client addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is.', + 'Some event client addresses are defined ambiguously. The event\nwill sometimes list an IP, a domain or a unix socket. You should always store\nthe raw address in the `.address` field.\n\nThen it should be duplicated to `.ip` or `.domain`, depending on which one\nit is.', }, { - name: 'ip', - level: 'core', - type: 'ip', - description: 'IP address of the client. Can be one or multiple IPv4 or IPv6 addresses.', + name: 'as.number', + level: 'extended', + type: 'long', + description: + 'Unique number allocated to the autonomous system. The autonomous\nsystem number (ASN) uniquely identifies each network on the Internet.', + example: 15169, }, { - name: 'port', + name: 'as.organization.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Organization name.', + example: 'Google LLC', + }, + { + name: 'bytes', level: 'core', type: 'long', - description: 'Port of the client.', + format: 'bytes', + description: 'Bytes sent from the client to the server.', + example: 184, }, { - name: 'mac', + name: 'domain', level: 'core', type: 'keyword', - description: 'MAC address of the client.', + ignore_above: 1024, + description: 'Client domain.', }, { - name: 'domain', + name: 'geo.city_name', level: 'core', type: 'keyword', - description: 'Client domain.', + ignore_above: 1024, + description: 'City name.', + example: 'Montreal', }, { - name: 'bytes', + name: 'geo.continent_name', level: 'core', - type: 'long', - format: 'bytes', - example: 184, - description: 'Bytes sent from the client to the server.', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the continent.', + example: 'North America', }, { - name: 'packets', + name: 'geo.country_iso_code', level: 'core', - type: 'long', - example: 12, - description: 'Packets sent from the client to the server.', + type: 'keyword', + ignore_above: 1024, + description: 'Country ISO code.', + example: 'CA', }, { - name: 'geo', - title: 'Geo', - group: 2, - description: - 'Geo fields can carry data about a specific location related to an event or geo information derived from an IP field.', - type: 'group', - fields: [ - { - name: 'location', - level: 'core', - type: 'geo_point', - description: 'Longitude and latitude.', - example: '{ "lon": -73.614830, "lat": 45.505918 }', - }, - { - name: 'continent_name', - level: 'core', - type: 'keyword', - description: 'Name of the continent.', - example: 'North America', - }, - { - name: 'country_name', - level: 'core', - type: 'keyword', - description: 'Country name.', - example: 'Canada', - }, - { - name: 'region_name', - level: 'core', - type: 'keyword', - description: 'Region name.', - example: 'Quebec', - }, - { - name: 'city_name', - level: 'core', - type: 'keyword', - description: 'City name.', - example: 'Montreal', - }, - { - name: 'country_iso_code', - level: 'core', - type: 'keyword', - description: 'Country ISO code.', - example: 'CA', - }, - { - name: 'region_iso_code', - level: 'core', - type: 'keyword', - description: 'Region ISO code.', - example: 'CA-QC', - }, - { - name: 'name', - level: 'extended', - type: 'keyword', - description: - 'User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation.', - example: 'boston-dc', - }, - ], + name: 'geo.country_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Country name.', + example: 'Canada', }, - ], - }, - { - name: 'cloud', - title: 'Cloud', - group: 2, - description: 'Fields related to the cloud or infrastructure the events are coming from.', - footnote: - 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.', - type: 'group', - fields: [ { - name: 'provider', + name: 'geo.location', + level: 'core', + type: 'geo_point', + description: 'Longitude and latitude.', + example: '{ "lon": -73.614830, "lat": 45.505918 }', + }, + { + name: 'geo.name', level: 'extended', - example: 'ec2', type: 'keyword', + ignore_above: 1024, description: - 'Name of the cloud provider. Example values are ec2, gce, or digitalocean.', + 'User-defined description of a location, at the level of granularity\nthey care about.\n\nCould be the name of their data centers, the floor number, if this describes\na local physical entity, city names.\n\nNot typically used in automated geolocation.', + example: 'boston-dc', }, { - name: 'availability_zone', - level: 'extended', - example: 'us-east-1c', + name: 'geo.region_iso_code', + level: 'core', type: 'keyword', - description: 'Availability zone in which this host is running.', + ignore_above: 1024, + description: 'Region ISO code.', + example: 'CA-QC', }, { - name: 'region', - level: 'extended', + name: 'geo.region_name', + level: 'core', type: 'keyword', - example: 'us-east-1', - description: 'Region in which this host is running.', + ignore_above: 1024, + description: 'Region name.', + example: 'Quebec', }, { - name: 'instance.id', - level: 'extended', - type: 'keyword', - example: 'i-1234567890abcdef0', - description: 'Instance ID of the host machine.', + name: 'ip', + level: 'core', + type: 'ip', + description: + 'IP address of the client.\n\nCan be one or multiple IPv4 or IPv6 addresses.', }, { - name: 'instance.name', - level: 'extended', + name: 'mac', + level: 'core', type: 'keyword', - description: 'Instance name of the host machine.', + ignore_above: 1024, + description: 'MAC address of the client.', }, { - name: 'machine.type', + name: 'nat.ip', level: 'extended', - type: 'keyword', - example: 't2.medium', - description: 'Machine type of the host machine.', + type: 'ip', + description: + 'Translated IP of source based NAT sessions (e.g. internal client\nto internet).\n\nTypically connections traversing load balancers, firewalls, or routers.', }, { - name: 'account.id', + name: 'nat.port', level: 'extended', - type: 'keyword', - example: 666777888999, + type: 'long', + format: 'string', description: - 'The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.', + 'Translated port of source based NAT sessions (e.g. internal client\nto internet).\n\nTypically connections traversing load balancers, firewalls, or routers.', }, - ], - }, - { - name: 'container', - title: 'Container', - group: 2, - description: - 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.', - type: 'group', - fields: [ { - name: 'runtime', - level: 'extended', - type: 'keyword', - description: 'Runtime managing this container.', - example: 'docker', + name: 'packets', + level: 'core', + type: 'long', + description: 'Packets sent from the client to the server.', + example: 12, }, { - name: 'id', + name: 'port', level: 'core', + type: 'long', + format: 'string', + description: 'Port of the client.', + }, + { + name: 'registered_domain', + level: 'extended', type: 'keyword', - description: 'Unique container id.', + ignore_above: 1024, + description: + 'The highest registered client domain, stripped of the subdomain.\n\nFor example, the registered domain for "foo.google.com" is "google.com".\n\nThis value can be determined precisely with a list like the public suffix\nlist (http://publicsuffix.org). Trying to approximate this by simply taking\nthe last two labels will not work well for TLDs such as "co.uk".', + example: 'google.com', }, { - name: 'image.name', + name: 'top_level_domain', level: 'extended', type: 'keyword', - description: 'Name of the image the container was built on.', + ignore_above: 1024, + description: + 'The effective top level domain (eTLD), also known as the domain\nsuffix, is the last part of the domain name. For example, the top level domain\nfor google.com is "com".\n\nThis value can be determined precisely with a list like the public suffix\nlist (http://publicsuffix.org). Trying to approximate this by simply taking\nthe last label will not work well for effective TLDs such as "co.uk".', + example: 'co.uk', }, { - name: 'image.tag', + name: 'user.domain', level: 'extended', type: 'keyword', - description: 'Container image tag.', + ignore_above: 1024, + description: + 'Name of the directory the user is a member of.\n\nFor example, an LDAP or Active Directory domain name.', }, { - name: 'name', + name: 'user.email', level: 'extended', type: 'keyword', - description: 'Container name.', + ignore_above: 1024, + description: 'User email address.', }, { - name: 'labels', + name: 'user.full_name', level: 'extended', - type: 'object', - object_type: 'keyword', - description: 'Image labels.', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: "User's full name, if available.", + example: 'Albert Einstein', }, - ], - }, - { - name: 'destination', - title: 'Destination', - group: 2, - description: - 'Destination fields describe details about the destination of a packet/event. Destination fields are usually populated in conjunction with source fields.', - type: 'group', - fields: [ { - name: 'address', + name: 'user.group.domain', level: 'extended', type: 'keyword', + ignore_above: 1024, description: - 'Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is.', + 'Name of the directory the group is a member of.\n\nFor example, an LDAP or Active Directory domain name.', }, { - name: 'ip', - level: 'core', - type: 'ip', - description: - 'IP address of the destination. Can be one or multiple IPv4 or IPv6 addresses.', + name: 'user.group.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifier for the group on the system/platform.', }, { - name: 'port', - level: 'core', - type: 'long', - description: 'Port of the destination.', + name: 'user.group.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the group.', }, { - name: 'mac', - level: 'core', + name: 'user.hash', + level: 'extended', type: 'keyword', - description: 'MAC address of the destination.', + ignore_above: 1024, + description: + 'Unique user hash to correlate information for a user in anonymized\nform.\n\nUseful if `user.id` or `user.name` contain confidential information and cannot\nbe used.', }, { - name: 'domain', + name: 'user.id', level: 'core', type: 'keyword', - description: 'Destination domain.', + ignore_above: 1024, + description: 'Unique identifiers of the user.', }, { - name: 'bytes', + name: 'user.name', level: 'core', - type: 'long', - format: 'bytes', - example: 184, - description: 'Bytes sent from the destination to the source.', - }, - { - name: 'packets', - level: 'core', - type: 'long', - example: 12, - description: 'Packets sent from the destination to the source.', - }, - { - name: 'geo', - title: 'Geo', - group: 2, - description: - 'Geo fields can carry data about a specific location related to an event or geo information derived from an IP field.', - type: 'group', - fields: [ - { - name: 'location', - level: 'core', - type: 'geo_point', - description: 'Longitude and latitude.', - example: '{ "lon": -73.614830, "lat": 45.505918 }', - }, - { - name: 'continent_name', - level: 'core', - type: 'keyword', - description: 'Name of the continent.', - example: 'North America', - }, - { - name: 'country_name', - level: 'core', - type: 'keyword', - description: 'Country name.', - example: 'Canada', - }, - { - name: 'region_name', - level: 'core', - type: 'keyword', - description: 'Region name.', - example: 'Quebec', - }, - { - name: 'city_name', - level: 'core', - type: 'keyword', - description: 'City name.', - example: 'Montreal', - }, - { - name: 'country_iso_code', - level: 'core', - type: 'keyword', - description: 'Country ISO code.', - example: 'CA', - }, - { - name: 'region_iso_code', - level: 'core', - type: 'keyword', - description: 'Region ISO code.', - example: 'CA-QC', - }, + type: 'keyword', + ignore_above: 1024, + multi_fields: [ { - name: 'name', - level: 'extended', - type: 'keyword', - description: - 'User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation.', - example: 'boston-dc', + name: 'text', + type: 'text', + norms: false, + default_field: false, }, ], + description: 'Short name or login of the user.', + example: 'albert', }, ], }, { - name: 'ecs', - title: 'ECS', + name: 'cloud', + title: 'Cloud', group: 2, - description: 'Meta-information specific to ECS.', + description: 'Fields related to the cloud or infrastructure the events are coming\nfrom.', + footnote: + 'Examples: If Metricbeat is running on an EC2 host and fetches data\nfrom its host, the cloud info contains the data about this machine. If Metricbeat\nruns on a remote machine outside the cloud and fetches data from a service running\nin the cloud, the field contains cloud data from the machine the service is\nrunning on.', type: 'group', fields: [ { - name: 'version', - level: 'core', + name: 'account.id', + level: 'extended', type: 'keyword', - required: true, + ignore_above: 1024, description: - 'ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. The current version is 1.0.0-beta2 .', - example: '1.0.0-beta2', + 'The cloud account or organization id used to identify different\nentities in a multi-tenant environment.\n\nExamples: AWS account id, Google Cloud ORG Id, or other unique identifier.', + example: 666777888999, }, - ], - }, - { - name: 'error', - title: 'Error', - group: 2, - description: - 'These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error.', - type: 'group', - fields: [ { - name: 'id', - level: 'core', + name: 'availability_zone', + level: 'extended', type: 'keyword', - description: 'Unique identifier for the error.', + ignore_above: 1024, + description: 'Availability zone in which this host is running.', + example: 'us-east-1c', }, { - name: 'message', - level: 'core', - type: 'text', - description: 'Error message.', + name: 'instance.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Instance ID of the host machine.', + example: 'i-1234567890abcdef0', }, { - name: 'code', - level: 'core', + name: 'instance.name', + level: 'extended', type: 'keyword', - description: 'Error code describing the error.', + ignore_above: 1024, + description: 'Instance name of the host machine.', }, - ], - }, - { - name: 'event', - title: 'Event', - group: 2, - description: - 'The event fields are used for context information about the log or metric event itself. A log is defined as an event containing details of something that happened. Log events must include the time at which the thing happened. Examples of log events include a process starting on a host, a network packet being sent from a source to a destination, or a network connection between a client and a server being initiated or closed. A metric is defined as an event containing one or more numerical or categorical measurements and the time at which the measurement was taken. Examples of metric events include memory pressure measured on a host, or vulnerabilities measured on a scanned host.', - type: 'group', - fields: [ { - name: 'id', - level: 'core', + name: 'machine.type', + level: 'extended', type: 'keyword', - description: 'Unique ID to describe the event.', - example: '8a4f500d', + ignore_above: 1024, + description: 'Machine type of the host machine.', + example: 't2.medium', }, { - name: 'kind', + name: 'provider', level: 'extended', type: 'keyword', + ignore_above: 1024, description: - 'The kind of the event. This gives information about what type of information the event contains, without being specific to the contents of the event. Examples are `event`, `state`, `alarm`. Warning: In future versions of ECS, we plan to provide a list of acceptable values for this field, please use with caution.', - example: 'state', + 'Name of the cloud provider. Example values are aws, azure, gcp,\nor digitalocean.', + example: 'aws', }, { - name: 'category', - level: 'core', + name: 'region', + level: 'extended', type: 'keyword', - description: - 'Event category. This contains high-level information about the contents of the event. It is more generic than `event.action`, in the sense that typically a category contains multiple actions. Warning: In future versions of ECS, we plan to provide a list of acceptable values for this field, please use with caution.', - example: 'user-management', + ignore_above: 1024, + description: 'Region in which this host is running.', + example: 'us-east-1', }, + ], + }, + { + name: 'code_signature', + title: 'Code Signature', + group: 2, + description: 'These fields contain information about binary code signatures.', + type: 'group', + fields: [ { - name: 'action', + name: 'exists', level: 'core', - type: 'keyword', - description: - 'The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer.', - example: 'user-password-change', + type: 'boolean', + description: 'Boolean to capture if a signature is present.', + example: 'true', + default_field: false, }, { - name: 'outcome', + name: 'status', level: 'extended', type: 'keyword', + ignore_above: 1024, description: - 'The outcome of the event. If the event describes an action, this fields contains the outcome of that action. Examples outcomes are `success` and `failure`. Warning: In future versions of ECS, we plan to provide a list of acceptable values for this field, please use with caution.', - example: 'success', - }, - { - name: 'type', - level: 'core', - type: 'keyword', - description: 'Reserved for future usage. Please avoid using this field for user data.', + 'Additional information about the certificate status.\n\nThis is useful for logging cryptographic errors with the certificate validity\nor trust status. Leave unpopulated if the validity or trust of the certificate\nwas unchecked.', + example: 'ERROR_UNTRUSTED_ROOT', + default_field: false, }, { - name: 'module', + name: 'subject_name', level: 'core', type: 'keyword', - description: - 'Name of the module this data is coming from. This information is coming from the modules used in Beats or Logstash.', - example: 'mysql', + ignore_above: 1024, + description: 'Subject name of the code signer', + example: 'Microsoft Corporation', + default_field: false, }, { - name: 'dataset', - level: 'core', - type: 'keyword', + name: 'trusted', + level: 'extended', + type: 'boolean', description: - 'Name of the dataset. The concept of a `dataset` (fileset / metricset) is used in Beats as a subset of modules. It contains the information which is currently stored in metricset.name and metricset.module or fileset.name.', - example: 'stats', + 'Stores the trust status of the certificate chain.\n\nValidating the trust of the certificate chain may be complicated, and this\nfield should only be populated by tools that actively check the status.', + example: 'true', + default_field: false, }, { - name: 'severity', - level: 'core', - type: 'long', - example: '7', + name: 'valid', + level: 'extended', + type: 'boolean', description: - "Severity describes the severity of the event. What the different severity values mean can very different between use cases. It's up to the implementer to make sure severities are consistent across events. ", + 'Boolean to capture if the digital signature is verified against\nthe binary content.\n\nLeave unpopulated if a certificate was unchecked.', + example: 'true', + default_field: false, }, + ], + }, + { + name: 'container', + title: 'Container', + group: 2, + description: + 'Container fields are used for meta information about the specific\ncontainer that is the source of information.\n\nThese fields help correlate data based containers from any runtime.', + type: 'group', + fields: [ { - name: 'original', + name: 'id', level: 'core', type: 'keyword', - example: - 'Sep 19 08:26:10 host CEF:0|Security| threatmanager|1.0|100| worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2spt=1232', - description: - 'Raw text message of entire event. Used to demonstrate log integrity. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`.', - index: false, - doc_values: false, + ignore_above: 1024, + description: 'Unique container id.', }, { - name: 'hash', + name: 'image.name', level: 'extended', type: 'keyword', - example: '123456789012345678901234567890ABCD', - description: - 'Hash (perhaps logstash fingerprint) of raw field to be able to demonstrate log integrity.', - }, - { - name: 'duration', - level: 'core', - type: 'long', - format: 'duration', - input_format: 'nanoseconds', - description: - 'Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time.', + ignore_above: 1024, + description: 'Name of the image the container was built on.', }, { - name: 'timezone', + name: 'image.tag', level: 'extended', type: 'keyword', - description: - 'This field should be populated when the event\'s timestamp does not include timezone information already (e.g. default Syslog timestamps). It\'s optional otherwise. Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00").', - }, - { - name: 'created', - level: 'core', - type: 'date', - description: - 'event.created contains the date when the event was created. This timestamp is distinct from @timestamp in that @timestamp contains the processed timestamp. For logs these two timestamps can be different as the timestamp in the log line and when the event is read for example by Filebeat are not identical. `@timestamp` must contain the timestamp extracted from the log line, event.created when the log line is read. The same could apply to package capturing where @timestamp contains the timestamp extracted from the network package and event.created when the event was created. In case the two timestamps are identical, @timestamp should be used.', + ignore_above: 1024, + description: 'Container image tags.', }, { - name: 'start', + name: 'labels', level: 'extended', - type: 'date', - description: - 'event.start contains the date when the event started or when the activity was first observed.', + type: 'object', + object_type: 'keyword', + description: 'Image labels.', }, { - name: 'end', + name: 'name', level: 'extended', - type: 'date', - description: - 'event.end contains the date when the event ended or when the activity was last observed.', - }, - { - name: 'risk_score', - level: 'core', - type: 'float', - description: - "Risk score or priority of the event (e.g. security solutions). Use your system's original value here. ", + type: 'keyword', + ignore_above: 1024, + description: 'Container name.', }, { - name: 'risk_score_norm', + name: 'runtime', level: 'extended', - type: 'float', - description: - 'Normalized risk score or priority of the event, on a scale of 0 to 100. This is mainly useful if you use more than one system that assigns risk scores, and you want to see a normalized value across all systems.', + type: 'keyword', + ignore_above: 1024, + description: 'Runtime managing this container.', + example: 'docker', }, ], }, { - name: 'file', + name: 'destination', + title: 'Destination', group: 2, - title: 'File', description: - 'A file is defined as a set of information that has been created on, or has existed on a filesystem. File objects can be associated with host events, network events, and/or file events (e.g., those produced by File Integrity Monitoring [FIM] products or services). File fields provide details about the affected file associated with the event or metric.', + 'Destination fields describe details about the destination of a packet/event.\n\nDestination fields are usually populated in conjunction with source fields.', type: 'group', fields: [ { - name: 'path', + name: 'address', level: 'extended', type: 'keyword', - description: 'Path to the file.', + ignore_above: 1024, + description: + 'Some event destination addresses are defined ambiguously. The\nevent will sometimes list an IP, a domain or a unix socket. You should always\nstore the raw address in the `.address` field.\n\nThen it should be duplicated to `.ip` or `.domain`, depending on which one\nit is.', }, { - name: 'target_path', + name: 'as.number', level: 'extended', - type: 'keyword', - description: 'Target path for symlinks.', + type: 'long', + description: + 'Unique number allocated to the autonomous system. The autonomous\nsystem number (ASN) uniquely identifies each network on the Internet.', + example: 15169, }, { - name: 'extension', + name: 'as.organization.name', level: 'extended', type: 'keyword', - description: 'File extension. This should allow easy filtering by file extensions.', - example: 'png', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Organization name.', + example: 'Google LLC', }, { - name: 'type', - level: 'extended', - type: 'keyword', - description: 'File type (file, dir, or symlink).', + name: 'bytes', + level: 'core', + type: 'long', + format: 'bytes', + description: 'Bytes sent from the destination to the source.', + example: 184, }, { - name: 'device', - level: 'extended', + name: 'domain', + level: 'core', type: 'keyword', - description: 'Device that is the source of the file.', + ignore_above: 1024, + description: 'Destination domain.', }, { - name: 'inode', - level: 'extended', + name: 'geo.city_name', + level: 'core', type: 'keyword', - description: 'Inode representing the file in the filesystem.', + ignore_above: 1024, + description: 'City name.', + example: 'Montreal', }, { - name: 'uid', - level: 'extended', + name: 'geo.continent_name', + level: 'core', type: 'keyword', - description: 'The user ID (UID) or security identifier (SID) of the file owner.', + ignore_above: 1024, + description: 'Name of the continent.', + example: 'North America', }, { - name: 'owner', - level: 'extended', + name: 'geo.country_iso_code', + level: 'core', type: 'keyword', - description: "File owner's username.", + ignore_above: 1024, + description: 'Country ISO code.', + example: 'CA', }, { - name: 'gid', - level: 'extended', + name: 'geo.country_name', + level: 'core', type: 'keyword', - description: 'Primary group ID (GID) of the file.', + ignore_above: 1024, + description: 'Country name.', + example: 'Canada', }, { - name: 'group', - level: 'extended', - type: 'keyword', - description: 'Primary group name of the file.', - }, - { - name: 'mode', - level: 'extended', - type: 'keyword', - example: 416, - description: 'Mode of the file in octal representation.', - }, - { - name: 'size', - level: 'extended', - type: 'long', - format: 'bytes', - description: 'File size in bytes (field is only added when `type` is `file`).', - }, - { - name: 'mtime', - level: 'extended', - type: 'date', - description: 'Last time file content was modified.', - }, - { - name: 'ctime', - level: 'extended', - type: 'date', - description: 'Last time file metadata changed.', - }, - ], - }, - { - name: 'group', - title: 'Group', - group: 2, - description: - 'The group fields are meant to represent groups that are relevant to the event.', - type: 'group', - fields: [ - { - name: 'id', - level: 'extended', - type: 'keyword', - description: 'Unique identifier for the group on the system/platform.', + name: 'geo.location', + level: 'core', + type: 'geo_point', + description: 'Longitude and latitude.', + example: '{ "lon": -73.614830, "lat": 45.505918 }', }, { - name: 'name', + name: 'geo.name', level: 'extended', type: 'keyword', - description: 'Name of the group.', - }, - ], - }, - { - name: 'host', - title: 'Host', - group: 2, - description: - 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or on which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.', - type: 'group', - fields: [ - { - name: 'hostname', - level: 'core', - type: 'keyword', + ignore_above: 1024, description: - 'Hostname of the host. It normally contains what the `hostname` command returns on the host machine.', + 'User-defined description of a location, at the level of granularity\nthey care about.\n\nCould be the name of their data centers, the floor number, if this describes\na local physical entity, city names.\n\nNot typically used in automated geolocation.', + example: 'boston-dc', }, { - name: 'name', + name: 'geo.region_iso_code', level: 'core', type: 'keyword', - description: - 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.', + ignore_above: 1024, + description: 'Region ISO code.', + example: 'CA-QC', }, { - name: 'id', + name: 'geo.region_name', level: 'core', type: 'keyword', - description: - 'Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.', + ignore_above: 1024, + description: 'Region name.', + example: 'Quebec', }, { name: 'ip', level: 'core', type: 'ip', - description: 'Host ip address.', + description: + 'IP address of the destination.\n\nCan be one or multiple IPv4 or IPv6 addresses.', }, { name: 'mac', level: 'core', type: 'keyword', - description: 'Host mac address.', + ignore_above: 1024, + description: 'MAC address of the destination.', }, { - name: 'type', - level: 'core', - type: 'keyword', + name: 'nat.ip', + level: 'extended', + type: 'ip', description: - 'Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.', + 'Translated ip of destination based NAT sessions (e.g. internet\nto private DMZ)\n\nTypically used with load balancers, firewalls, or routers.', }, { - name: 'architecture', - level: 'core', - type: 'keyword', - example: 'x86_64', - description: 'Operating system architecture.', + name: 'nat.port', + level: 'extended', + type: 'long', + format: 'string', + description: + 'Port the source session is translated to by NAT Device.\n\nTypically used with load balancers, firewalls, or routers.', }, { - name: 'os', - title: 'Operating System', - group: 2, - description: 'The OS fields contain information about the operating system.', - reusable: { - top_level: false, - expected: ['observer', 'host', 'user_agent'], - }, - type: 'group', - fields: [ - { - name: 'platform', - level: 'extended', - type: 'keyword', - description: 'Operating system platform (such centos, ubuntu, windows).', - example: 'darwin', - }, - { - name: 'name', - level: 'extended', - type: 'keyword', - example: 'Mac OS X', - description: 'Operating system name, without the version.', - }, - { - name: 'full', - level: 'extended', - type: 'keyword', - example: 'Mac OS Mojave', - description: 'Operating system name, including the version or code name.', - }, - { - name: 'family', - level: 'extended', - type: 'keyword', - example: 'debian', - description: 'OS family (such as redhat, debian, freebsd, windows).', - }, - { - name: 'version', - level: 'extended', - type: 'keyword', - example: '10.14.1', - description: 'Operating system version as a raw string.', - }, - { - name: 'kernel', - level: 'extended', - type: 'keyword', - example: '4.4.0-112-generic', - description: 'Operating system kernel version as a raw string.', - }, - ], + name: 'packets', + level: 'core', + type: 'long', + description: 'Packets sent from the destination to the source.', + example: 12, }, { - name: 'geo', - title: 'Geo', - group: 2, - description: - 'Geo fields can carry data about a specific location related to an event or geo information derived from an IP field.', - type: 'group', - fields: [ - { - name: 'location', - level: 'core', - type: 'geo_point', - description: 'Longitude and latitude.', - example: '{ "lon": -73.614830, "lat": 45.505918 }', - }, - { - name: 'continent_name', - level: 'core', - type: 'keyword', - description: 'Name of the continent.', - example: 'North America', - }, - { - name: 'country_name', - level: 'core', - type: 'keyword', - description: 'Country name.', - example: 'Canada', - }, - { - name: 'region_name', - level: 'core', - type: 'keyword', - description: 'Region name.', - example: 'Quebec', - }, - { - name: 'city_name', - level: 'core', - type: 'keyword', - description: 'City name.', - example: 'Montreal', - }, - { - name: 'country_iso_code', - level: 'core', - type: 'keyword', - description: 'Country ISO code.', - example: 'CA', - }, - { - name: 'region_iso_code', - level: 'core', - type: 'keyword', - description: 'Region ISO code.', - example: 'CA-QC', - }, - { - name: 'name', - level: 'extended', - type: 'keyword', - description: - 'User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation.', - example: 'boston-dc', - }, - ], + name: 'port', + level: 'core', + type: 'long', + format: 'string', + description: 'Port of the destination.', }, - ], - }, - { - name: 'http', - title: 'HTTP', - group: 2, - description: 'Fields related to HTTP activity.', - type: 'group', - fields: [ { - name: 'request.method', + name: 'registered_domain', level: 'extended', type: 'keyword', + ignore_above: 1024, description: - 'Http request method. The field value must be normalized to lowercase for querying. See "Lowercase Capitalization" in the "Implementing ECS" section.', - example: 'get, post, put', + 'The highest registered destination domain, stripped of the subdomain.\n\nFor example, the registered domain for "foo.google.com" is "google.com".\n\nThis value can be determined precisely with a list like the public suffix\nlist (http://publicsuffix.org). Trying to approximate this by simply taking\nthe last two labels will not work well for TLDs such as "co.uk".', + example: 'google.com', }, { - name: 'request.body.content', + name: 'top_level_domain', level: 'extended', type: 'keyword', - description: 'The full http request body.', - example: 'Hello world', + ignore_above: 1024, + description: + 'The effective top level domain (eTLD), also known as the domain\nsuffix, is the last part of the domain name. For example, the top level domain\nfor google.com is "com".\n\nThis value can be determined precisely with a list like the public suffix\nlist (http://publicsuffix.org). Trying to approximate this by simply taking\nthe last label will not work well for effective TLDs such as "co.uk".', + example: 'co.uk', }, { - name: 'request.referrer', + name: 'user.domain', level: 'extended', type: 'keyword', - description: 'Referrer for this HTTP request.', - example: 'https://blog.example.com/', - }, - { - name: 'response.status_code', - level: 'extended', - type: 'long', - description: 'Http response status code.', - example: 404, + ignore_above: 1024, + description: + 'Name of the directory the user is a member of.\n\nFor example, an LDAP or Active Directory domain name.', }, { - name: 'response.body.content', + name: 'user.email', level: 'extended', type: 'keyword', - description: 'The full http response body.', - example: 'Hello world', + ignore_above: 1024, + description: 'User email address.', }, { - name: 'version', + name: 'user.full_name', level: 'extended', type: 'keyword', - description: 'Http version.', - example: 1.1, + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: "User's full name, if available.", + example: 'Albert Einstein', }, { - name: 'request.bytes', + name: 'user.group.domain', level: 'extended', - type: 'long', - format: 'bytes', - description: 'Total size in bytes of the request (body and headers).', - example: 1437, + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the directory the group is a member of.\n\nFor example, an LDAP or Active Directory domain name.', }, { - name: 'request.body.bytes', + name: 'user.group.id', level: 'extended', - type: 'long', - format: 'bytes', - description: 'Size in bytes of the request body.', - example: 887, + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifier for the group on the system/platform.', }, { - name: 'response.bytes', + name: 'user.group.name', level: 'extended', - type: 'long', - format: 'bytes', - description: 'Total size in bytes of the response (body and headers).', - example: 1437, + type: 'keyword', + ignore_above: 1024, + description: 'Name of the group.', }, { - name: 'response.body.bytes', + name: 'user.hash', level: 'extended', - type: 'long', - format: 'bytes', - description: 'Size in bytes of the response body.', - example: 887, + type: 'keyword', + ignore_above: 1024, + description: + 'Unique user hash to correlate information for a user in anonymized\nform.\n\nUseful if `user.id` or `user.name` contain confidential information and cannot\nbe used.', }, - ], - }, - { - name: 'log', - title: 'Log', - description: 'Fields which are specific to log events.', - type: 'group', - fields: [ { - name: 'level', + name: 'user.id', level: 'core', type: 'keyword', - description: 'Log level of the log event. Some examples are `WARN`, `ERR`, `INFO`.', - example: 'ERR', + ignore_above: 1024, + description: 'Unique identifiers of the user.', }, { - name: 'original', + name: 'user.name', level: 'core', type: 'keyword', - example: 'Sep 19 08:26:10 localhost My log', - index: false, - doc_values: false, - description: - " This is the original log message and contains the full log message before splitting it up in multiple parts. In contrast to the `message` field which can contain an extracted part of the log message, this field contains the original, full log message. It can have already some modifications applied like encoding or new lines removed to clean up the log message. This field is not indexed and doc_values are disabled so it can't be queried but the value can be retrieved from `_source`. ", + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Short name or login of the user.', + example: 'albert', }, ], }, { - name: 'network', - title: 'Network', + name: 'dll', + title: 'DLL', group: 2, description: - 'The network is defined as the communication path over which a host or network event happens. The network.* fields should be populated with details about the network activity associated with an event.', + 'These fields contain information about code libraries dynamically\nloaded into processes.\n\n\nMany operating systems refer to "shared code libraries" with different names,\nbut this field set refers to all of the following:\n\n* Dynamic-link library (`.dll`) commonly used on Windows\n\n* Shared Object (`.so`) commonly used on Unix-like operating systems\n\n* Dynamic library (`.dylib`) commonly used on macOS', type: 'group', fields: [ { - name: 'name', + name: 'code_signature.exists', + level: 'core', + type: 'boolean', + description: 'Boolean to capture if a signature is present.', + example: 'true', + default_field: false, + }, + { + name: 'code_signature.status', level: 'extended', type: 'keyword', - description: 'Name given by operators to sections of their network.', - example: 'Guest Wifi', + ignore_above: 1024, + description: + 'Additional information about the certificate status.\n\nThis is useful for logging cryptographic errors with the certificate validity\nor trust status. Leave unpopulated if the validity or trust of the certificate\nwas unchecked.', + example: 'ERROR_UNTRUSTED_ROOT', + default_field: false, }, { - name: 'type', + name: 'code_signature.subject_name', level: 'core', type: 'keyword', + ignore_above: 1024, + description: 'Subject name of the code signer', + example: 'Microsoft Corporation', + default_field: false, + }, + { + name: 'code_signature.trusted', + level: 'extended', + type: 'boolean', description: - 'In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc The field value must be normalized to lowercase for querying. See "Lowercase Capitalization" in the "Implementing ECS" section.', - example: 'ipv4', + 'Stores the trust status of the certificate chain.\n\nValidating the trust of the certificate chain may be complicated, and this\nfield should only be populated by tools that actively check the status.', + example: 'true', + default_field: false, }, { - name: 'iana_number', + name: 'code_signature.valid', level: 'extended', - type: 'keyword', + type: 'boolean', description: - 'IANA Protocol Number (https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml). Standardized list of protocols. This aligns well with NetFlow and sFlow related logs which use the IANA Protocol Number.', - example: 6, + 'Boolean to capture if the digital signature is verified against\nthe binary content.\n\nLeave unpopulated if a certificate was unchecked.', + example: 'true', + default_field: false, }, { - name: 'transport', - level: 'core', + name: 'hash.md5', + level: 'extended', type: 'keyword', - description: - 'Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. See "Lowercase Capitalization" in the "Implementing ECS" section.', - example: 'tcp', + ignore_above: 1024, + description: 'MD5 hash.', + default_field: false, }, { - name: 'application', + name: 'hash.sha1', level: 'extended', type: 'keyword', - description: - 'A name given to an application. This can be arbitrarily assigned for things like microservices, but also apply to things like skype, icq, facebook, twitter. This would be used in situations where the vendor or service can be decoded such as from the source/dest IP owners, ports, or wire format. The field value must be normalized to lowercase for querying. See "Lowercase Capitalization" in the "Implementing ECS" section.', - example: 'aim', + ignore_above: 1024, + description: 'SHA1 hash.', + default_field: false, }, { - name: 'protocol', - level: 'core', + name: 'hash.sha256', + level: 'extended', type: 'keyword', - description: - 'L7 Network protocol name. ex. http, lumberjack, transport protocol. The field value must be normalized to lowercase for querying. See "Lowercase Capitalization" in the "Implementing ECS" section.', - example: 'http', + ignore_above: 1024, + description: 'SHA256 hash.', + default_field: false, }, { - name: 'direction', + name: 'hash.sha512', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'SHA512 hash.', + default_field: false, + }, + { + name: 'name', level: 'core', type: 'keyword', + ignore_above: 1024, description: - "Direction of the network traffic. Recommended values are: * inbound * outbound * internal * external * unknown When mapping events from a host-based monitoring context, populate this field from the host's point of view. When mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of your network perimeter. ", - example: 'inbound', + 'Name of the library.\n\nThis generally maps to the name of the file on disk.', + example: 'kernel32.dll', + default_field: false, }, { - name: 'forwarded_ip', - level: 'core', - type: 'ip', - description: 'Host IP address when the source IP address is the proxy.', - example: '192.1.1.2', - }, - { - name: 'community_id', + name: 'path', level: 'extended', type: 'keyword', - description: - 'A hash of source and destination IPs and ports, as well as the protocol used in a communication. This is a tool-agnostic standard to identify flows. Learn more at https://github.com/corelight/community-id-spec.', - example: '1:hO+sN4H+MG5MY/8hIrXPqc4ZQz0=', - }, - { - name: 'bytes', - level: 'core', - type: 'long', - format: 'bytes', - description: - 'Total bytes transferred in both directions. If `source.bytes` and `destination.bytes` are known, `network.bytes` is their sum.', - example: 368, - }, - { - name: 'packets', - level: 'core', - type: 'long', - description: - 'Total packets transferred in both directions. If `source.packets` and `destination.packets` are known, `network.packets` is their sum.', - example: 24, - }, - ], - }, - { - name: 'observer', - title: 'Observer', - group: 2, - description: - 'An observer is defined as a special network, security, or application device used to detect, observe, or create network, security, or application-related events and metrics. This could be a custom hardware appliance or a server that has been configured to run special network, security, or application software. Examples include firewalls, intrusion detection/prevention systems, network monitoring sensors, web application firewalls, data loss prevention systems, and APM servers. The observer.* fields shall be populated with details of the system, if any, that detects, observes and/or creates a network, security, or application event or metric. Message queues and ETL components used in processing events or metrics are not considered observers in ECS.', - type: 'group', - fields: [ - { - name: 'mac', - level: 'core', - type: 'keyword', - description: 'MAC address of the observer', - }, - { - name: 'ip', - level: 'core', - type: 'ip', - description: 'IP address of the observer.', + ignore_above: 1024, + description: 'Full file path of the library.', + example: 'C:\\Windows\\System32\\kernel32.dll', + default_field: false, }, { - name: 'hostname', - level: 'core', + name: 'pe.company', + level: 'extended', type: 'keyword', - description: 'Hostname of the observer.', + ignore_above: 1024, + description: 'Internal company name of the file, provided at compile-time.', + example: 'Microsoft Corporation', + default_field: false, }, { - name: 'vendor', - level: 'core', + name: 'pe.description', + level: 'extended', type: 'keyword', - description: 'observer vendor information.', + ignore_above: 1024, + description: 'Internal description of the file, provided at compile-time.', + example: 'Paint', + default_field: false, }, { - name: 'version', - level: 'core', + name: 'pe.file_version', + level: 'extended', type: 'keyword', - description: 'Observer version.', + ignore_above: 1024, + description: 'Internal version of the file, provided at compile-time.', + example: '6.3.9600.17415', + default_field: false, }, { - name: 'serial_number', + name: 'pe.original_file_name', level: 'extended', type: 'keyword', - description: 'Observer serial number.', + ignore_above: 1024, + description: 'Internal name of the file, provided at compile-time.', + example: 'MSPAINT.EXE', + default_field: false, }, { - name: 'type', - level: 'core', + name: 'pe.product', + level: 'extended', type: 'keyword', - description: - 'The type of the observer the data is coming from. There is no predefined list of observer types. Some examples are `forwarder`, `firewall`, `ids`, `ips`, `proxy`, `poller`, `sensor`, `APM server`.', - example: 'firewall', - }, - { - name: 'os', - title: 'Operating System', - group: 2, - description: 'The OS fields contain information about the operating system.', - reusable: { - top_level: false, - expected: ['observer', 'host', 'user_agent'], - }, - type: 'group', - fields: [ - { - name: 'platform', - level: 'extended', - type: 'keyword', - description: 'Operating system platform (such centos, ubuntu, windows).', - example: 'darwin', - }, - { - name: 'name', - level: 'extended', - type: 'keyword', - example: 'Mac OS X', - description: 'Operating system name, without the version.', - }, - { - name: 'full', - level: 'extended', - type: 'keyword', - example: 'Mac OS Mojave', - description: 'Operating system name, including the version or code name.', - }, - { - name: 'family', - level: 'extended', - type: 'keyword', - example: 'debian', - description: 'OS family (such as redhat, debian, freebsd, windows).', - }, - { - name: 'version', - level: 'extended', - type: 'keyword', - example: '10.14.1', - description: 'Operating system version as a raw string.', - }, - { - name: 'kernel', - level: 'extended', - type: 'keyword', - example: '4.4.0-112-generic', - description: 'Operating system kernel version as a raw string.', - }, - ], - }, - { - name: 'geo', - title: 'Geo', - group: 2, - description: - 'Geo fields can carry data about a specific location related to an event or geo information derived from an IP field.', - type: 'group', - fields: [ - { - name: 'location', - level: 'core', - type: 'geo_point', - description: 'Longitude and latitude.', - example: '{ "lon": -73.614830, "lat": 45.505918 }', - }, - { - name: 'continent_name', - level: 'core', - type: 'keyword', - description: 'Name of the continent.', - example: 'North America', - }, - { - name: 'country_name', - level: 'core', - type: 'keyword', - description: 'Country name.', - example: 'Canada', - }, - { - name: 'region_name', - level: 'core', - type: 'keyword', - description: 'Region name.', - example: 'Quebec', - }, - { - name: 'city_name', - level: 'core', - type: 'keyword', - description: 'City name.', - example: 'Montreal', - }, - { - name: 'country_iso_code', - level: 'core', - type: 'keyword', - description: 'Country ISO code.', - example: 'CA', - }, - { - name: 'region_iso_code', - level: 'core', - type: 'keyword', - description: 'Region ISO code.', - example: 'CA-QC', - }, - { - name: 'name', - level: 'extended', - type: 'keyword', - description: - 'User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation.', - example: 'boston-dc', - }, - ], + ignore_above: 1024, + description: 'Internal product name of the file, provided at compile-time.', + example: 'Microsoft® Windows® Operating System', + default_field: false, }, ], }, { - name: 'organization', - title: 'Organization', + name: 'dns', + title: 'DNS', group: 2, description: - 'The organization fields enrich data with information about the company or entity the data is associated with. These fields help you arrange or filter data stored in an index by one or multiple organizations.', + 'Fields describing DNS queries and answers.\n\nDNS events should either represent a single DNS query prior to getting answers\n(`dns.type:query`) or they should represent a full exchange and contain the\nquery details as well as all of the answers that were provided for this query\n(`dns.type:answer`).', type: 'group', fields: [ { - name: 'name', + name: 'answers', level: 'extended', - type: 'keyword', - description: 'Organization name.', + type: 'object', + object_type: 'keyword', + description: + 'An array containing an object for each answer section returned\nby the server.\n\nThe main keys that should be present in these objects are defined by ECS.\nRecords that have more information may contain more keys than what ECS defines.\n\nNot all DNS data sources give all details about DNS answers. At minimum, answer\nobjects must contain the `data` key. If more information is available, map\nas much of it to ECS as possible, and add any additional fields to the answer\nobjects as custom fields.', }, { - name: 'id', + name: 'answers.class', level: 'extended', type: 'keyword', - description: 'Unique identifier for the organization.', + ignore_above: 1024, + description: 'The class of DNS data contained in this resource record.', + example: 'IN', }, - ], - }, - { - name: 'os', - title: 'Operating System', - group: 2, - description: 'The OS fields contain information about the operating system.', - reusable: { - top_level: false, - expected: ['observer', 'host', 'user_agent'], - }, - type: 'group', - fields: [ { - name: 'platform', + name: 'answers.data', level: 'extended', type: 'keyword', - description: 'Operating system platform (such centos, ubuntu, windows).', - example: 'darwin', + ignore_above: 1024, + description: + 'The data describing the resource.\n\nThe meaning of this data depends on the type and class of the resource record.', + example: '10.10.10.10', }, { - name: 'name', + name: 'answers.name', level: 'extended', type: 'keyword', - example: 'Mac OS X', - description: 'Operating system name, without the version.', + ignore_above: 1024, + description: + 'The domain name to which this resource record pertains.\n\nIf a chain of CNAME is being resolved, each answer `name` should be the\none that corresponds with the answer `data`. It should not simply be the\noriginal `question.name` repeated.', + example: 'www.google.com', }, { - name: 'full', + name: 'answers.ttl', level: 'extended', - type: 'keyword', - example: 'Mac OS Mojave', - description: 'Operating system name, including the version or code name.', + type: 'long', + description: + 'The time interval in seconds that this resource record may be cached\nbefore it should be discarded. Zero values mean that the data should not be\ncached.', + example: 180, }, { - name: 'family', + name: 'answers.type', level: 'extended', type: 'keyword', - example: 'debian', - description: 'OS family (such as redhat, debian, freebsd, windows).', + ignore_above: 1024, + description: 'The type of data contained in this resource record.', + example: 'CNAME', }, { - name: 'version', + name: 'header_flags', level: 'extended', type: 'keyword', - example: '10.14.1', - description: 'Operating system version as a raw string.', + ignore_above: 1024, + description: + 'Array of 2 letter DNS header flags.\n\nExpected values are: AA, TC, RD, RA, AD, CD, DO.', + example: ['RD', 'RA'], }, { - name: 'kernel', + name: 'id', level: 'extended', type: 'keyword', - example: '4.4.0-112-generic', - description: 'Operating system kernel version as a raw string.', + ignore_above: 1024, + description: + 'The DNS packet identifier assigned by the program that generated\nthe query. The identifier is copied to the response.', + example: 62111, }, - ], - }, - { - name: 'process', - title: 'Process', - group: 2, - description: - 'These fields contain information about a process. These fields can help you correlate metrics information with a process id/name from a log message. The `process.pid` often stays in the metric itself and is copied to the global field for correlation.', - type: 'group', - fields: [ { - name: 'pid', - level: 'core', - type: 'long', - description: 'Process id.', - example: 'ssh', + name: 'op_code', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The DNS operation code that specifies the kind of query in the\nmessage. This value is set by the originator of a query and copied into the\nresponse.', + example: 'QUERY', }, { - name: 'name', + name: 'question.class', level: 'extended', type: 'keyword', - description: 'Process name. Sometimes called program name or similar.', - example: 'ssh', + ignore_above: 1024, + description: 'The class of records being queried.', + example: 'IN', }, { - name: 'ppid', + name: 'question.name', level: 'extended', - type: 'long', - description: 'Process parent id.', + type: 'keyword', + ignore_above: 1024, + description: + 'The name being queried.\n\nIf the name field contains non-printable characters (below 32 or above 126),\nthose characters should be represented as escaped base 10 integers (\\DDD).\nBack slashes and quotes should be escaped. Tabs, carriage returns, and line\nfeeds should be converted to \\t, \\r, and \\n respectively.', + example: 'www.google.com', }, { - name: 'args', + name: 'question.registered_domain', level: 'extended', type: 'keyword', - description: 'Process arguments. May be filtered to protect sensitive information.', - example: ['ssh', '-l', 'user', '10.0.0.16'], + ignore_above: 1024, + description: + 'The highest registered domain, stripped of the subdomain.\n\nFor example, the registered domain for "foo.google.com" is "google.com".\n\nThis value can be determined precisely with a list like the public suffix\nlist (http://publicsuffix.org). Trying to approximate this by simply taking\nthe last two labels will not work well for TLDs such as "co.uk".', + example: 'google.com', }, { - name: 'executable', + name: 'question.subdomain', level: 'extended', type: 'keyword', - description: 'Absolute path to the process executable.', - example: '/usr/bin/ssh', + ignore_above: 1024, + description: + 'The subdomain is all of the labels under the registered_domain.\n\nIf the domain has multiple levels of subdomain, such as "sub2.sub1.example.com",\nthe subdomain field should contain "sub2.sub1", with no trailing period.', + example: 'www', }, { - name: 'title', + name: 'question.top_level_domain', level: 'extended', type: 'keyword', + ignore_above: 1024, description: - 'Process title. The proctitle, some times the same as process name. Can also be different: for example a browser setting its title to the web page currently opened.', + 'The effective top level domain (eTLD), also known as the domain\nsuffix, is the last part of the domain name. For example, the top level domain\nfor google.com is "com".\n\nThis value can be determined precisely with a list like the public suffix\nlist (http://publicsuffix.org). Trying to approximate this by simply taking\nthe last label will not work well for effective TLDs such as "co.uk".', + example: 'co.uk', }, { - name: 'thread.id', + name: 'question.type', level: 'extended', - type: 'long', - example: 4242, - description: 'Thread ID.', + type: 'keyword', + ignore_above: 1024, + description: 'The type of record being queried.', + example: 'AAAA', }, { - name: 'start', + name: 'resolved_ip', level: 'extended', - type: 'date', - example: '2016-05-23T08:05:34.853Z', - description: 'The time the process started.', + type: 'ip', + description: + 'Array containing all IPs seen in `answers.data`.\n\nThe `answers` array can be difficult to use, because of the variety of data\nformats it can contain. Extracting all IP addresses seen in there to `dns.resolved_ip`\nmakes it possible to index them as IP addresses, and makes them easier to\nvisualize and query for.', + example: ['10.10.10.10', '10.10.10.11'], }, { - name: 'working_directory', + name: 'response_code', level: 'extended', type: 'keyword', - example: '/home/alice', - description: 'The working directory of the process.', + ignore_above: 1024, + description: 'The DNS response code.', + example: 'NOERROR', + }, + { + name: 'type', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The type of DNS event captured, query or answer.\n\nIf your source of DNS events only gives you DNS queries, you should only create\ndns events of type `dns.type:query`.\n\nIf your source of DNS events gives you answers as well, you should create\none event per query (optionally as soon as the query is seen). And a second\nevent containing all query details as well as an array of answers.', + example: 'answer', }, ], }, { - name: 'related', - title: 'Related', + name: 'ecs', + title: 'ECS', group: 2, - description: - 'This field set is meant to facilitate pivoting around a piece of data. Some pieces of information can be seen in many places in ECS. To facilitate searching for them, append values to their corresponding field in `related.`. A concrete example is IP addresses, which can be under host, observer, source, destination, client, server, and network.forwarded_ip. If you append all IPs to `related.ip`, you can then search for a given IP trivially, no matter where it appeared, by querying `related.ip:a.b.c.d`.', + description: 'Meta-information specific to ECS.', type: 'group', fields: [ { - name: 'ip', - level: 'extended', - type: 'ip', - description: 'All of the IPs seen on your event.', + name: 'version', + level: 'core', + required: true, + type: 'keyword', + ignore_above: 1024, + description: + 'ECS version this event conforms to. `ecs.version` is a required\nfield and must exist in all events.\n\nWhen querying across multiple indices -- which may conform to slightly different\nECS versions -- this field lets integrations adjust to the schema version\nof the events.', + example: '1.0.0', }, ], }, { - name: 'server', - title: 'Server', + name: 'error', + title: 'Error', group: 2, description: - 'A Server is defined as the responder in a network connection for events regarding sessions, connections, or bidirectional flow records. For TCP events, the server is the receiver of the initial SYN packet(s) of the TCP connection. For other protocols, the server is generally the responder in the network transaction. Some systems actually use the term "responder" to refer the server in TCP connections. The server fields describe details about the system acting as the server in the network event. Server fields are usually populated in conjunction with client fields. Server fields are generally not populated for packet-level events. Client / server representations can add semantic context to an exchange, which is helpful to visualize the data in certain situations. If your context falls in that category, you should still ensure that source and destination are filled appropriately.', + 'These fields can represent errors of any kind.\n\nUse them for errors that happen while fetching events or in cases where the\nevent itself contains an error.', type: 'group', fields: [ { - name: 'address', - level: 'extended', - type: 'keyword', - description: - 'Some event server addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is.', - }, - { - name: 'ip', - level: 'core', - type: 'ip', - description: 'IP address of the server. Can be one or multiple IPv4 or IPv6 addresses.', - }, - { - name: 'port', - level: 'core', - type: 'long', - description: 'Port of the server.', - }, - { - name: 'mac', + name: 'code', level: 'core', type: 'keyword', - description: 'MAC address of the server.', + ignore_above: 1024, + description: 'Error code describing the error.', }, { - name: 'domain', + name: 'id', level: 'core', type: 'keyword', - description: 'Server domain.', - }, - { - name: 'bytes', - level: 'core', - type: 'long', - format: 'bytes', - example: 184, - description: 'Bytes sent from the server to the client.', + ignore_above: 1024, + description: 'Unique identifier for the error.', }, { - name: 'packets', + name: 'message', level: 'core', - type: 'long', - example: 12, - description: 'Packets sent from the server to the client.', + type: 'text', + description: 'Error message.', }, { - name: 'geo', - title: 'Geo', - group: 2, - description: - 'Geo fields can carry data about a specific location related to an event or geo information derived from an IP field.', - type: 'group', - fields: [ - { - name: 'location', - level: 'core', - type: 'geo_point', - description: 'Longitude and latitude.', - example: '{ "lon": -73.614830, "lat": 45.505918 }', - }, - { - name: 'continent_name', - level: 'core', - type: 'keyword', - description: 'Name of the continent.', - example: 'North America', - }, - { - name: 'country_name', - level: 'core', - type: 'keyword', - description: 'Country name.', - example: 'Canada', - }, - { - name: 'region_name', - level: 'core', - type: 'keyword', - description: 'Region name.', - example: 'Quebec', - }, - { - name: 'city_name', - level: 'core', - type: 'keyword', - description: 'City name.', - example: 'Montreal', - }, - { - name: 'country_iso_code', - level: 'core', - type: 'keyword', - description: 'Country ISO code.', - example: 'CA', - }, - { - name: 'region_iso_code', - level: 'core', - type: 'keyword', - description: 'Region ISO code.', - example: 'CA-QC', - }, + name: 'stack_trace', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ { - name: 'name', - level: 'extended', - type: 'keyword', - description: - 'User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation.', - example: 'boston-dc', + name: 'text', + type: 'text', + norms: false, + default_field: false, }, ], + description: 'The stack trace of this error in plain text.', + }, + { + name: 'type', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'The type of the error, for example the class name of the exception.', + example: 'java.lang.NullPointerException', }, ], }, { - name: 'service', - title: 'Service', + name: 'event', + title: 'Event', group: 2, description: - 'The service fields describe the service for or from which the data was collected. These fields help you find and correlate logs for a specific service and version.', + 'The event fields are used for context information about the log\nor metric event itself.\n\nA log is defined as an event containing details of something that happened.\nLog events must include the time at which the thing happened. Examples of log\nevents include a process starting on a host, a network packet being sent from\na source to a destination, or a network connection between a client and a server\nbeing initiated or closed. A metric is defined as an event containing one or\nmore numerical measurements and the time at which the measurement was taken.\nExamples of metric events include memory pressure measured on a host and device\ntemperature. See the `event.kind` definition in this section for additional\ndetails about metric and state events.', type: 'group', fields: [ { - name: 'id', + name: 'action', level: 'core', type: 'keyword', + ignore_above: 1024, description: - 'Unique identifier of the running service. This id should uniquely identify this service. This makes it possible to correlate logs and metrics for one specific service. Example: If you are experiencing issues with one redis instance, you can filter on that id to see metrics and logs for that single instance.', - example: 'd37e5ebfe0ae6c4972dbe9f0174a1637bb8247f6', + 'The action captured by the event.\n\nThis describes the information in the event. It is more specific than `event.category`.\nExamples are `group-add`, `process-started`, `file-created`. The value is\nnormally defined by the implementer.', + example: 'user-password-change', }, { - name: 'name', + name: 'category', level: 'core', type: 'keyword', - example: 'elasticsearch-metrics', + ignore_above: 1024, description: - 'Name of the service data is collected from. The name of the service is normally user given. This allows if two instances of the same service are running on the same machine they can be differentiated by the `service.name`. Also it allows for distributed services that run on multiple hosts to correlate the related instances based on the name. In the case of Elasticsearch the service.name could contain the cluster name. For Beats the service.name is by default a copy of the `service.type` field if no name is specified.', + 'This is one of four ECS Categorization Fields, and indicates the\nsecond level in the ECS category hierarchy.\n\n`event.category` represents the "big buckets" of ECS categories. For example,\nfiltering on `event.category:process` yields all events relating to process\nactivity. This field is closely related to `event.type`, which is used as\na subcategory.\n\nThis field is an array. This will allow proper categorization of some events\nthat fall in multiple categories.', + example: 'authentication', }, { - name: 'type', - level: 'core', + name: 'code', + level: 'extended', type: 'keyword', - example: 'elasticsearch', + ignore_above: 1024, description: - 'The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`.', + 'Identification code for this event, if one exists.\n\nSome event sources use event codes to identify messages unambiguously, regardless\nof message language or wording adjustments over time. An example of this is\nthe Windows Event ID.', + example: 4648, }, { - name: 'state', + name: 'created', level: 'core', - type: 'keyword', - description: 'Current state of the service.', + type: 'date', + description: + 'event.created contains the date/time when the event was first\nread by an agent, or by your pipeline.\n\nThis field is distinct from @timestamp in that @timestamp typically contain\nthe time extracted from the original event.\n\nIn most situations, these two timestamps will be slightly different. The difference\ncan be used to calculate the delay between your source generating an event,\nand the time when your agent first processed it. This can be used to monitor\nyour agent or pipeline ability to keep up with your event source.\n\nIn case the two timestamps are identical, @timestamp should be used.', + example: '2016-05-23T08:05:34.857Z', }, { - name: 'version', + name: 'dataset', level: 'core', type: 'keyword', - example: '3.2.4', + ignore_above: 1024, description: - 'Version of the service the data was collected from. This allows to look at a data set only for a specific version of a service.', + 'Name of the dataset.\n\nIf an event source publishes more than one type of log or events (e.g. access\nlog, error log), the dataset is used to specify which one the event comes\nfrom.\n\nIt is recommended but not required to start the dataset name with the module\nname, followed by a dot, then the dataset name.', + example: 'apache.access', }, { - name: 'ephemeral_id', + name: 'duration', + level: 'core', + type: 'long', + format: 'duration', + input_format: 'nanoseconds', + output_format: 'asMilliseconds', + output_precision: 1, + description: + 'Duration of the event in nanoseconds.\n\nIf event.start and event.end are known this value should be the difference\nbetween the end and start time.', + }, + { + name: 'end', level: 'extended', - type: 'keyword', + type: 'date', description: - 'Ephemeral identifier of this service (if one exists). This id normally changes across restarts, but `service.id` does not.', - example: '8a4f500f', + 'event.end contains the date when the event ended or when the activity\nwas last observed.', }, - ], - }, - { - name: 'source', - title: 'Source', - group: 2, - description: - 'Source fields describe details about the source of a packet/event. Source fields are usually populated in conjunction with destination fields.', - type: 'group', - fields: [ { - name: 'address', + name: 'hash', level: 'extended', type: 'keyword', + ignore_above: 1024, description: - 'Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is.', + 'Hash (perhaps logstash fingerprint) of raw field to be able to\ndemonstrate log integrity.', + example: '123456789012345678901234567890ABCD', }, { - name: 'ip', + name: 'id', level: 'core', - type: 'ip', - description: 'IP address of the source. Can be one or multiple IPv4 or IPv6 addresses.', + type: 'keyword', + ignore_above: 1024, + description: 'Unique ID to describe the event.', + example: '8a4f500d', }, { - name: 'port', + name: 'ingested', level: 'core', - type: 'long', - description: 'Port of the source.', + type: 'date', + description: + 'Timestamp when an event arrived in the central data store.\n\nThis is different from `@timestamp`, which is when the event originally occurred. It is\nalso different from `event.created`, which is meant to capture the first time\nan agent saw the event.\n\nIn normal conditions, assuming no tampering, the timestamps should chronologically\nlook like this: `@timestamp` < `event.created` < `event.ingested`.', + example: '2016-05-23T08:05:35.101Z', + default_field: false, }, { - name: 'mac', + name: 'kind', level: 'core', type: 'keyword', - description: 'MAC address of the source.', + ignore_above: 1024, + description: + 'This is one of four ECS Categorization Fields, and indicates the\nhighest level in the ECS category hierarchy.\n\n`event.kind` gives high-level information about what type of information the\nevent contains, without being specific to the contents of the event. For example,\nvalues of this field distinguish alert events from metric events.\n\nThe value of this field can be used to inform how these kinds of events should\nbe handled. They may warrant different retention, different access control,\nit may also help understand whether the data coming in at a regular interval\nor not.', + example: 'alert', }, { - name: 'domain', + name: 'module', level: 'core', type: 'keyword', - description: 'Source domain.', + ignore_above: 1024, + description: + 'Name of the module this data is coming from.\n\nIf your monitoring agent supports the concept of modules or plugins to process\nevents of a given source (e.g. Apache logs), `event.module` should contain\nthe name of this module.', + example: 'apache', }, { - name: 'bytes', + name: 'original', level: 'core', - type: 'long', - format: 'bytes', - example: 184, - description: 'Bytes sent from the source to the destination.', + type: 'keyword', + ignore_above: 1024, + description: + 'Raw text message of entire event. Used to demonstrate log integrity.\n\nThis field is not indexed and doc_values are disabled. It cannot be searched,\nbut it can be retrieved from `_source`.', + example: + 'Sep 19 08:26:10 host CEF:0|Security| threatmanager|1.0|100|\nworm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2spt=1232', }, { - name: 'packets', + name: 'outcome', level: 'core', - type: 'long', - example: 12, - description: 'Packets sent from the source to the destination.', - }, - { - name: 'geo', - title: 'Geo', - group: 2, + type: 'keyword', + ignore_above: 1024, description: - 'Geo fields can carry data about a specific location related to an event or geo information derived from an IP field.', - type: 'group', - fields: [ - { - name: 'location', - level: 'core', - type: 'geo_point', - description: 'Longitude and latitude.', - example: '{ "lon": -73.614830, "lat": 45.505918 }', - }, - { - name: 'continent_name', - level: 'core', - type: 'keyword', - description: 'Name of the continent.', - example: 'North America', - }, - { - name: 'country_name', - level: 'core', - type: 'keyword', - description: 'Country name.', - example: 'Canada', - }, - { - name: 'region_name', - level: 'core', - type: 'keyword', - description: 'Region name.', - example: 'Quebec', - }, - { - name: 'city_name', - level: 'core', - type: 'keyword', - description: 'City name.', - example: 'Montreal', - }, - { - name: 'country_iso_code', - level: 'core', - type: 'keyword', - description: 'Country ISO code.', - example: 'CA', - }, - { - name: 'region_iso_code', - level: 'core', - type: 'keyword', - description: 'Region ISO code.', - example: 'CA-QC', - }, - { - name: 'name', - level: 'extended', - type: 'keyword', - description: - 'User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation.', - example: 'boston-dc', - }, - ], + 'This is one of four ECS Categorization Fields, and indicates the\nlowest level in the ECS category hierarchy.\n\n`event.outcome` simply denotes whether the event represents a success or a\nfailure from the perspective of the entity that produced the event.\n\nNote that when a single transaction is described in multiple events, each\nevent may populate different values of `event.outcome`, according to their\nperspective.\n\nAlso note that in the case of a compound event (a single event that contains\nmultiple logical events), this field should be populated with the value that\nbest captures the overall success or failure from the perspective of the event\nproducer.\n\nFurther note that not all events will have an associated outcome. For example,\nthis field is generally not populated for metric events, events with `event.type:info`,\nor any events for which an outcome does not make logical sense.', + example: 'success', }, - ], - }, - { - name: 'url', - title: 'URL', - description: 'URL fields provide a complete URL, with scheme, host, and path.', - type: 'group', - fields: [ { - name: 'original', + name: 'provider', level: 'extended', type: 'keyword', + ignore_above: 1024, description: - 'Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not.', - example: - 'https://www.elastic.co:443/search?q=elasticsearch#top or /search?q=elasticsearch', + 'Source of the event.\n\nEvent transports such as Syslog or the Windows Event Log typically mention\nthe source of an event. It can be the name of the software that generated\nthe event (e.g. Sysmon, httpd), or of a subsystem of the operating system\n(kernel, Microsoft-Windows-Security-Auditing).', + example: 'kernel', }, { - name: 'full', + name: 'reference', level: 'extended', type: 'keyword', + ignore_above: 1024, description: - 'If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source.', - example: 'https://www.elastic.co:443/search?q=elasticsearch#top', + 'Reference URL linking to additional information about this event.\n\nThis URL links to a static definition of the this event. Alert events, indicated\nby `event.kind:alert`, are a common use case for this field.', + example: 'https://system.vendor.com/event/#0001234', + default_field: false, }, { - name: 'scheme', - level: 'extended', - type: 'keyword', + name: 'risk_score', + level: 'core', + type: 'float', description: - 'Scheme of the request, such as "https". Note: The `:` is not part of the scheme.', - example: 'https', + "Risk score or priority of the event (e.g. security solutions).\nUse your system's original value here.", }, { - name: 'domain', + name: 'risk_score_norm', level: 'extended', - type: 'keyword', + type: 'float', description: - 'Domain of the request, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field.', - example: 'www.elastic.co', + 'Normalized risk score or priority of the event, on a scale of\n0 to 100.\n\nThis is mainly useful if you use more than one system that assigns risk scores,\nand you want to see a normalized value across all systems.', }, { - name: 'port', + name: 'sequence', level: 'extended', - type: 'integer', - description: 'Port of the request, such as 443.', - example: 443, + type: 'long', + format: 'string', + description: + 'Sequence number of the event.\n\nThe sequence number is a value published by some event sources, to make the\nexact ordering of events unambiguous, regardless of the timestamp precision.', }, { - name: 'path', - level: 'extended', - type: 'keyword', - description: 'Path of the request, such as "/search".', + name: 'severity', + level: 'core', + type: 'long', + format: 'string', + description: + 'The numeric severity of the event according to your event source.\n\nWhat the different severity values mean can be different between sources and\nuse cases. It is up to the implementer to make sure severities are consistent\nacross events from the same source.\n\nThe Syslog severity belongs in `log.syslog.severity.code`. `event.severity`\nis meant to represent the severity according to the event source (e.g. firewall,\nIDS). If the event source does not publish its own severity, you may optionally\ncopy the `log.syslog.severity.code` to `event.severity`.', + example: 7, }, { - name: 'query', + name: 'start', level: 'extended', - type: 'keyword', + type: 'date', description: - 'The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases.', + 'event.start contains the date when the event started or when the\nactivity was first observed.', }, { - name: 'fragment', + name: 'timezone', level: 'extended', type: 'keyword', + ignore_above: 1024, description: - 'Portion of the url after the `#`, such as "top". The `#` is not part of the fragment.', + 'This field should be populated when the event timestamp does\nnot include timezone information already (e.g. default Syslog timestamps).\nIt is optional otherwise.\n\nAcceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"),\nabbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00").', }, { - name: 'username', - level: 'extended', + name: 'type', + level: 'core', type: 'keyword', - description: 'Username of the request.', + ignore_above: 1024, + description: + 'This is one of four ECS Categorization Fields, and indicates the\nthird level in the ECS category hierarchy.\n\n`event.type` represents a categorization "sub-bucket" that, when used along\nwith the `event.category` field values, enables filtering events down to a\nlevel appropriate for single visualization.\n\nThis field is an array. This will allow proper categorization of some events\nthat fall in multiple event types.', }, { - name: 'password', + name: 'url', level: 'extended', type: 'keyword', - description: 'Password of the request.', + ignore_above: 1024, + description: + 'URL linking to an external system to continue investigation of\nthis event.\n\nThis URL links to another system where in-depth investigation of the specific\noccurence of this event can take place. Alert events, indicated by `event.kind:alert`,\nare a common use case for this field.', + example: 'https://mysystem.mydomain.com/alert/5271dedb-f5b0-4218-87f0-4ac4870a38fe', + default_field: false, }, ], }, { - name: 'user', - title: 'User', + name: 'file', + title: 'File', group: 2, description: - 'The user fields describe information about the user that is relevant to the event. Fields can have one entry or multiple entries. If a user has more than one id, provide an array that includes all of them.', - reusable: { - top_level: true, - expected: ['client', 'destination', 'host', 'server', 'source'], - }, + 'A file is defined as a set of information that has been created\non, or has existed on a filesystem.\n\nFile objects can be associated with host events, network events, and/or file\nevents (e.g., those produced by File Integrity Monitoring [FIM] products or\nservices). File fields provide details about the affected file associated with\nthe event or metric.', type: 'group', fields: [ { - name: 'id', - level: 'core', + name: 'accessed', + level: 'extended', + type: 'date', + description: + 'Last time the file was accessed.\n\nNote that not all filesystems keep track of access time.', + }, + { + name: 'attributes', + level: 'extended', type: 'keyword', - description: 'One or multiple unique identifiers of the user.', + ignore_above: 1024, + description: + 'Array of file attributes.\n\nAttributes names will vary by platform. Here is a non-exhaustive list of values\nthat are expected in this field: archive, compressed, directory, encrypted,\nexecute, hidden, read, readonly, system, write.', + example: '["readonly", "system"]', + default_field: false, }, { - name: 'name', + name: 'code_signature.exists', level: 'core', - type: 'keyword', - example: 'albert', - description: 'Short name or login of the user.', + type: 'boolean', + description: 'Boolean to capture if a signature is present.', + example: 'true', + default_field: false, }, { - name: 'full_name', + name: 'code_signature.status', level: 'extended', type: 'keyword', - example: 'Albert Einstein', - description: "User's full name, if available. ", + ignore_above: 1024, + description: + 'Additional information about the certificate status.\n\nThis is useful for logging cryptographic errors with the certificate validity\nor trust status. Leave unpopulated if the validity or trust of the certificate\nwas unchecked.', + example: 'ERROR_UNTRUSTED_ROOT', + default_field: false, }, { - name: 'email', - level: 'extended', + name: 'code_signature.subject_name', + level: 'core', type: 'keyword', - description: 'User email address.', + ignore_above: 1024, + description: 'Subject name of the code signer', + example: 'Microsoft Corporation', + default_field: false, }, { - name: 'hash', + name: 'code_signature.trusted', level: 'extended', - type: 'keyword', + type: 'boolean', description: - 'Unique user hash to correlate information for a user in anonymized form. Useful if `user.id` or `user.name` contain confidential information and cannot be used.', + 'Stores the trust status of the certificate chain.\n\nValidating the trust of the certificate chain may be complicated, and this\nfield should only be populated by tools that actively check the status.', + example: 'true', + default_field: false, }, { - name: 'group', - title: 'Group', - group: 2, + name: 'code_signature.valid', + level: 'extended', + type: 'boolean', description: - 'The group fields are meant to represent groups that are relevant to the event.', - type: 'group', - fields: [ - { - name: 'id', - level: 'extended', - type: 'keyword', - description: 'Unique identifier for the group on the system/platform.', - }, - { - name: 'name', - level: 'extended', - type: 'keyword', - description: 'Name of the group.', - }, - ], + 'Boolean to capture if the digital signature is verified against\nthe binary content.\n\nLeave unpopulated if a certificate was unchecked.', + example: 'true', + default_field: false, }, - ], - }, - { - name: 'user_agent', - title: 'User agent', - group: 2, - description: - 'The user_agent fields normally come from a browser request. They often show up in web service logs coming from the parsed user agent string.', - type: 'group', - fields: [ { - name: 'original', + name: 'created', level: 'extended', - type: 'keyword', - description: 'Unparsed version of the user_agent.', - example: - 'Mozilla/5.0 (iPhone; CPU iPhone OS 12_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1', + type: 'date', + description: + 'File creation time.\n\nNote that not all filesystems store the creation time.', }, { - name: 'name', + name: 'ctime', level: 'extended', - type: 'keyword', - example: 'Safari', - description: 'Name of the user agent.', + type: 'date', + description: + 'Last time the file attributes or metadata changed.\n\nNote that changes to the file content will update `mtime`. This implies `ctime`\nwill be adjusted at the same time, since `mtime` is an attribute of the file.', }, { - name: 'version', + name: 'device', level: 'extended', type: 'keyword', - description: 'Version of the user agent.', - example: 12, + ignore_above: 1024, + description: 'Device that is the source of the file.', + example: 'sda', }, { - name: 'device.name', + name: 'directory', level: 'extended', type: 'keyword', - example: 'iPhone', - description: 'Name of the device.', + ignore_above: 1024, + description: + 'Directory where the file is located. It should include the drive\nletter, when appropriate.', + example: '/home/alice', }, { - name: 'os', - title: 'Operating System', - group: 2, - description: 'The OS fields contain information about the operating system.', - reusable: { - top_level: false, - expected: ['observer', 'host', 'user_agent'], - }, - type: 'group', - fields: [ - { - name: 'platform', - level: 'extended', - type: 'keyword', - description: 'Operating system platform (such centos, ubuntu, windows).', - example: 'darwin', - }, - { - name: 'name', - level: 'extended', - type: 'keyword', - example: 'Mac OS X', - description: 'Operating system name, without the version.', - }, - { - name: 'full', - level: 'extended', - type: 'keyword', - example: 'Mac OS Mojave', - description: 'Operating system name, including the version or code name.', - }, - { - name: 'family', - level: 'extended', - type: 'keyword', - example: 'debian', - description: 'OS family (such as redhat, debian, freebsd, windows).', - }, - { - name: 'version', - level: 'extended', - type: 'keyword', - example: '10.14.1', - description: 'Operating system version as a raw string.', - }, - { - name: 'kernel', - level: 'extended', - type: 'keyword', - example: '4.4.0-112-generic', - description: 'Operating system kernel version as a raw string.', - }, - ], + name: 'drive_letter', + level: 'extended', + type: 'keyword', + ignore_above: 1, + description: + 'Drive letter where the file is located. This field is only relevant\non Windows.\n\nThe value should be uppercase, and not include the colon.', + example: 'C', + default_field: false, }, - ], - }, - { - name: 'agent.hostname', - type: 'keyword', - description: 'Hostname of the agent.', - }, - ], - }, - { - key: 'beat', - title: 'Beat', - description: 'Contains common beat fields available in all event types.', - fields: [ - { - name: 'beat.timezone', - type: 'alias', - path: 'event.timezone', - migration: true, - }, - { - name: 'fields', - type: 'object', - object_type: 'keyword', - description: 'Contains user configurable fields.', - }, - { - name: 'error', - type: 'group', - description: 'Error fields containing additional info in case of errors.', - fields: [ { - name: 'type', + name: 'extension', + level: 'extended', type: 'keyword', - description: 'Error type.', + ignore_above: 1024, + description: 'File extension.', + example: 'png', }, - ], - }, - { - name: 'beat.name', - type: 'alias', - path: 'host.name', - migration: true, - }, - { - name: 'beat.hostname', - type: 'alias', - path: 'agent.hostname', - migration: true, - }, - ], - }, - { - key: 'cloud', - title: 'Cloud provider metadata', - description: 'Metadata from cloud providers added by the add_cloud_metadata processor.', - fields: [ - { - name: 'cloud.project.id', - example: 'project-x', - description: 'Name of the project in Google Cloud.', - }, - { - name: 'meta.cloud.provider', - type: 'alias', - path: 'cloud.provider', - migration: true, - }, - { - name: 'meta.cloud.instance_id', - type: 'alias', - path: 'cloud.instance.id', - migration: true, - }, - { - name: 'meta.cloud.instance_name', - type: 'alias', - path: 'cloud.instance.name', - migration: true, - }, - { - name: 'meta.cloud.machine_type', - type: 'alias', - path: 'cloud.machine.type', - migration: true, - }, - { - name: 'meta.cloud.availability_zone', - type: 'alias', - path: 'cloud.availability_zone', - migration: true, - }, - { - name: 'meta.cloud.project_id', - type: 'alias', - path: 'cloud.project.id', - migration: true, - }, - { - name: 'meta.cloud.region', - type: 'alias', - path: 'cloud.region', - migration: true, - }, - ], - }, - { - key: 'docker', - title: 'Docker', - description: 'Docker stats collected from Docker.', - short_config: false, - anchor: 'docker-processor', - fields: [ - { - name: 'docker', - type: 'group', - fields: [ { - name: 'container.id', - type: 'alias', - path: 'container.id', - migration: true, + name: 'gid', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Primary group ID (GID) of the file.', + example: '1001', }, { - name: 'container.image', - type: 'alias', - path: 'container.image.name', - migration: true, + name: 'group', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Primary group name of the file.', + example: 'alice', }, { - name: 'container.name', - type: 'alias', - path: 'container.name', - migration: true, + name: 'hash.md5', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'MD5 hash.', }, { - name: 'container.labels', - type: 'object', - object_type: 'keyword', - description: 'Image labels.', + name: 'hash.sha1', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'SHA1 hash.', }, - ], - }, - ], - }, - { - key: 'host', - title: 'Host', - description: 'Info collected for the host machine.', - anchor: 'host-processor', - }, - { - key: 'kubernetes', - title: 'Kubernetes', - description: 'Kubernetes metadata added by the kubernetes processor', - short_config: false, - anchor: 'kubernetes-processor', - fields: [ - { - name: 'kubernetes', - type: 'group', - fields: [ { - name: 'pod.name', + name: 'hash.sha256', + level: 'extended', type: 'keyword', - description: 'Kubernetes pod name', + ignore_above: 1024, + description: 'SHA256 hash.', }, { - name: 'pod.uid', + name: 'hash.sha512', + level: 'extended', type: 'keyword', - description: 'Kubernetes Pod UID', + ignore_above: 1024, + description: 'SHA512 hash.', }, { - name: 'namespace', + name: 'inode', + level: 'extended', type: 'keyword', - description: 'Kubernetes namespace', + ignore_above: 1024, + description: 'Inode representing the file in the filesystem.', + example: '256383', }, { - name: 'node.name', + name: 'mime_type', + level: 'extended', type: 'keyword', - description: 'Kubernetes node name', + ignore_above: 1024, + description: + 'MIME type should identify the format of the file or stream of bytes\nusing https://www.iana.org/assignments/media-types/media-types.xhtml[IANA\nofficial types], where possible. When more than one type is applicable, the\nmost specific type should be used.', + default_field: false, }, { - name: 'labels', - type: 'object', - description: 'Kubernetes labels map', + name: 'mode', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Mode of the file in octal representation.', + example: '0640', }, { - name: 'annotations', - type: 'object', - description: 'Kubernetes annotations map', + name: 'mtime', + level: 'extended', + type: 'date', + description: 'Last time the file content was modified.', }, { - name: 'container.name', + name: 'name', + level: 'extended', type: 'keyword', - description: 'Kubernetes container name', + ignore_above: 1024, + description: 'Name of the file including the extension, without the directory.', + example: 'example.png', }, { - name: 'container.image', + name: 'owner', + level: 'extended', type: 'keyword', - description: 'Kubernetes container image', - }, - ], - }, - ], - }, - { - key: 'process', - title: 'Process', - description: 'Process metadata fields', - fields: [ - { - name: 'process', - type: 'group', - fields: [ - { - name: 'exe', - type: 'alias', - path: 'process.executable', - migration: true, - }, - ], - }, - ], - }, - { - key: 'common', - title: 'Common', - description: - 'These fields contain data about the environment in which the transaction or flow was captured.', - fields: [ - { - name: 'type', - description: - 'The type of the transaction (for example, HTTP, MySQL, Redis, or RUM) or "flow" in case of flows.', - required: true, - }, - { - name: 'server.process.name', - description: 'The name of the process that served the transaction.', - }, - { - name: 'server.process.args', - description: 'The command-line of the process that served the transaction.', - }, - { - name: 'server.process.executable', - description: 'Absolute path to the server process executable.', - }, - { - name: 'server.process.working_directory', - description: 'The working directory of the server process.', - }, - { - name: 'server.process.start', - description: 'The time the server process started.', - }, - { - name: 'client.process.name', - description: 'The name of the process that initiated the transaction.', - }, - { - name: 'client.process.args', - description: 'The command-line of the process that initiated the transaction.', - }, - { - name: 'client.process.executable', - description: 'Absolute path to the client process executable.', - }, - { - name: 'client.process.working_directory', - description: 'The working directory of the client process.', - }, - { - name: 'client.process.start', - description: 'The time the client process started.', - }, - { - name: 'real_ip', - type: 'alias', - path: 'network.forwarded_ip', - migration: true, - description: - 'If the server initiating the transaction is a proxy, this field contains the original client IP address. For HTTP, for example, the IP address extracted from a configurable HTTP header, by default `X-Forwarded-For`. Unless this field is disabled, it always has a value, and it matches the `client_ip` for non proxy clients.', - }, - { - name: 'transport', - type: 'alias', - path: 'network.transport', - migration: true, - description: - 'The transport protocol used for the transaction. If not specified, then tcp is assumed.', - }, - ], - }, - { - key: 'flows_event', - title: 'Flow Event', - description: 'These fields contain data about the flow itself.', - fields: [ - { - name: 'flow.final', - type: 'boolean', - description: - 'Indicates if event is last event in flow. If final is false, the event reports an intermediate flow state only.', - }, - { - name: 'flow.id', - description: 'Internal flow ID based on connection meta data and address.', - }, - { - name: 'flow.vlan', - type: 'long', - description: - "VLAN identifier from the 802.1q frame. In case of a multi-tagged frame this field will be an array with the outer tag's VLAN identifier listed first. ", - }, - { - name: 'flow_id', - type: 'alias', - path: 'flow.id', - migration: true, - }, - { - name: 'final', - type: 'alias', - path: 'flow.final', - migration: true, - }, - { - name: 'vlan', - type: 'alias', - path: 'flow.vlan', - migration: true, - }, - { - name: 'source.stats.net_bytes_total', - type: 'alias', - path: 'source.bytes', - migration: true, - }, - { - name: 'source.stats.net_packets_total', - type: 'alias', - path: 'source.packets', - migration: true, - }, - { - name: 'dest.stats.net_bytes_total', - type: 'alias', - path: 'destination.bytes', - migration: true, - }, - { - name: 'dest.stats.net_packets_total', - type: 'alias', - path: 'destination.packets', - migration: true, - }, - ], - }, - { - key: 'trans_event', - title: 'Transaction Event', - description: 'These fields contain data about the transaction itself.', - fields: [ - { - name: 'status', - description: - 'The high level status of the transaction. The way to compute this value depends on the protocol, but the result has a meaning independent of the protocol.', - required: true, - possible_values: ['OK', 'Error', 'Server Error', 'Client Error'], - }, - { - name: 'method', - description: - 'The command/verb/method of the transaction. For HTTP, this is the method name (GET, POST, PUT, and so on), for SQL this is the verb (SELECT, UPDATE, DELETE, and so on).', - }, - { - name: 'resource', - description: - 'The logical resource that this transaction refers to. For HTTP, this is the URL path up to the last slash (/). For example, if the URL is `/users/1`, the resource is `/users`. For databases, the resource is typically the table name. The field is not filled for all transaction types.', - }, - { - name: 'path', - required: true, - description: - 'The path the transaction refers to. For HTTP, this is the URL. For SQL databases, this is the table name. For key-value stores, this is the key.', - }, - { - name: 'query', - type: 'keyword', - description: - 'The query in a human readable format. For HTTP, it will typically be something like `GET /users/_search?name=test`. For MySQL, it is something like `SELECT id from users where name=test`.', - }, - { - name: 'params', - type: 'text', - description: - 'The request parameters. For HTTP, these are the POST or GET parameters. For Thrift-RPC, these are the parameters from the request.', - }, - { - name: 'notes', - type: 'alias', - path: 'error.message', - description: - 'Messages from Packetbeat itself. This field usually contains error messages for interpreting the raw data. This information can be helpful for troubleshooting.', - }, - ], - }, - { - key: 'raw', - title: 'Raw', - description: 'These fields contain the raw transaction data.', - fields: [ - { - name: 'request', - type: 'text', - description: - 'For text protocols, this is the request as seen on the wire (application layer only). For binary protocols this is our representation of the request.', - }, - { - name: 'response', - type: 'text', - description: - 'For text protocols, this is the response as seen on the wire (application layer only). For binary protocols this is our representation of the request.', - }, - ], - }, - { - key: 'trans_measurements', - title: 'Measurements (Transactions)', - description: 'These fields contain measurements related to the transaction.', - fields: [ - { - name: 'bytes_in', - type: 'alias', - path: 'source.bytes', - description: - 'The number of bytes of the request. Note that this size is the application layer message length, without the length of the IP or TCP headers.', - }, - { - name: 'bytes_out', - type: 'alias', - path: 'destination.bytes', - description: - 'The number of bytes of the response. Note that this size is the application layer message length, without the length of the IP or TCP headers.', - }, - ], - }, - { - key: 'amqp', - title: 'AMQP', - description: 'AMQP specific event fields.', - fields: [ - { - name: 'amqp', - type: 'group', - fields: [ - { - name: 'reply-code', - type: 'long', - description: 'AMQP reply code to an error, similar to http reply-code', - example: 404, + ignore_above: 1024, + description: "File owner's username.", + example: 'alice', }, { - name: 'reply-text', + name: 'path', + level: 'extended', type: 'keyword', - description: 'Text explaining the error.', - }, - { - name: 'class-id', - type: 'long', - description: 'Failing method class.', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: + 'Full path to the file, including the file name. It should include\nthe drive letter, when appropriate.', + example: '/home/alice/example.png', }, { - name: 'method-id', - type: 'long', - description: 'Failing method ID.', + name: 'pe.company', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal company name of the file, provided at compile-time.', + example: 'Microsoft Corporation', + default_field: false, }, { - name: 'exchange', + name: 'pe.description', + level: 'extended', type: 'keyword', - description: 'Name of the exchange.', + ignore_above: 1024, + description: 'Internal description of the file, provided at compile-time.', + example: 'Paint', + default_field: false, }, { - name: 'exchange-type', + name: 'pe.file_version', + level: 'extended', type: 'keyword', - description: 'Exchange type.', - example: 'fanout', + ignore_above: 1024, + description: 'Internal version of the file, provided at compile-time.', + example: '6.3.9600.17415', + default_field: false, }, { - name: 'passive', - type: 'boolean', - description: 'If set, do not create exchange/queue.', + name: 'pe.original_file_name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal name of the file, provided at compile-time.', + example: 'MSPAINT.EXE', + default_field: false, }, { - name: 'durable', - type: 'boolean', - description: 'If set, request a durable exchange/queue.', + name: 'pe.product', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal product name of the file, provided at compile-time.', + example: 'Microsoft® Windows® Operating System', + default_field: false, }, { - name: 'exclusive', - type: 'boolean', - description: 'If set, request an exclusive queue.', + name: 'size', + level: 'extended', + type: 'long', + description: 'File size in bytes.\n\nOnly relevant when `file.type` is "file".', + example: 16384, }, { - name: 'auto-delete', - type: 'boolean', - description: 'If set, auto-delete queue when unused.', + name: 'target_path', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Target path for symlinks.', }, { - name: 'no-wait', - type: 'boolean', - description: 'If set, the server will not respond to the method.', + name: 'type', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'File type (file, dir, or symlink).', + example: 'file', }, { - name: 'consumer-tag', - description: 'Identifier for the consumer, valid within the current channel.', + name: 'uid', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'The user ID (UID) or security identifier (SID) of the file owner.', + example: '1001', }, + ], + }, + { + name: 'geo', + title: 'Geo', + group: 2, + description: + 'Geo fields can carry data about a specific location related to an\nevent.\n\nThis geolocation information can be derived from techniques such as Geo IP,\nor be user-supplied.', + type: 'group', + fields: [ { - name: 'delivery-tag', - type: 'long', - description: 'The server-assigned and channel-specific delivery tag.', + name: 'city_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'City name.', + example: 'Montreal', }, { - name: 'message-count', - type: 'long', - description: - 'The number of messages in the queue, which will be zero for newly-declared queues.', + name: 'continent_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the continent.', + example: 'North America', }, { - name: 'consumer-count', - type: 'long', - description: 'The number of consumers of a queue.', + name: 'country_iso_code', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Country ISO code.', + example: 'CA', }, { - name: 'routing-key', + name: 'country_name', + level: 'core', type: 'keyword', - description: 'Message routing key.', + ignore_above: 1024, + description: 'Country name.', + example: 'Canada', }, { - name: 'no-ack', - type: 'boolean', - description: 'If set, the server does not expect acknowledgements for messages.', + name: 'location', + level: 'core', + type: 'geo_point', + description: 'Longitude and latitude.', + example: '{ "lon": -73.614830, "lat": 45.505918 }', }, { - name: 'no-local', - type: 'boolean', + name: 'name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, description: - 'If set, the server will not send messages to the connection that published them.', + 'User-defined description of a location, at the level of granularity\nthey care about.\n\nCould be the name of their data centers, the floor number, if this describes\na local physical entity, city names.\n\nNot typically used in automated geolocation.', + example: 'boston-dc', }, { - name: 'if-unused', - type: 'boolean', - description: 'Delete only if unused.', + name: 'region_iso_code', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Region ISO code.', + example: 'CA-QC', }, { - name: 'if-empty', - type: 'boolean', - description: 'Delete only if empty.', + name: 'region_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Region name.', + example: 'Quebec', }, + ], + }, + { + name: 'group', + title: 'Group', + group: 2, + description: + 'The group fields are meant to represent groups that are relevant\nto the event.', + type: 'group', + fields: [ { - name: 'queue', + name: 'domain', + level: 'extended', type: 'keyword', - description: 'The queue name identifies the queue within the vhost.', - }, - { - name: 'redelivered', - type: 'boolean', - description: - 'Indicates that the message has been previously delivered to this or another client.', - }, - { - name: 'multiple', - type: 'boolean', - description: 'Acknowledge multiple messages.', - }, - { - name: 'arguments', - type: 'object', + ignore_above: 1024, description: - 'Optional additional arguments passed to some methods. Can be of various types.', - }, - { - name: 'mandatory', - type: 'boolean', - description: 'Indicates mandatory routing.', + 'Name of the directory the group is a member of.\n\nFor example, an LDAP or Active Directory domain name.', }, { - name: 'immediate', - type: 'boolean', - description: 'Request immediate delivery.', - }, - { - name: 'content-type', + name: 'id', + level: 'extended', type: 'keyword', - description: 'MIME content type.', - example: 'text/plain', + ignore_above: 1024, + description: 'Unique identifier for the group on the system/platform.', }, { - name: 'content-encoding', + name: 'name', + level: 'extended', type: 'keyword', - description: 'MIME content encoding.', - }, - { - name: 'headers', - type: 'object', - object_type: 'keyword', - description: 'Message header field table.', + ignore_above: 1024, + description: 'Name of the group.', }, + ], + }, + { + name: 'hash', + title: 'Hash', + group: 2, + description: + 'The hash fields represent different hash algorithms and their values.\n\nField names for common hashes (e.g. MD5, SHA1) are predefined. Add fields for\nother hashes by lowercasing the hash algorithm name and using underscore separators\nas appropriate (snake case, e.g. sha3_512).', + type: 'group', + fields: [ { - name: 'delivery-mode', + name: 'md5', + level: 'extended', type: 'keyword', - description: 'Non-persistent (1) or persistent (2).', - }, - { - name: 'priority', - type: 'long', - description: 'Message priority, 0 to 9.', + ignore_above: 1024, + description: 'MD5 hash.', }, { - name: 'correlation-id', + name: 'sha1', + level: 'extended', type: 'keyword', - description: 'Application correlation identifier.', + ignore_above: 1024, + description: 'SHA1 hash.', }, { - name: 'reply-to', + name: 'sha256', + level: 'extended', type: 'keyword', - description: 'Address to reply to.', + ignore_above: 1024, + description: 'SHA256 hash.', }, { - name: 'expiration', + name: 'sha512', + level: 'extended', type: 'keyword', - description: 'Message expiration specification.', + ignore_above: 1024, + description: 'SHA512 hash.', }, + ], + }, + { + name: 'host', + title: 'Host', + group: 2, + description: + 'A host is defined as a general computing instance.\n\nECS host.* fields should be populated with details about the host on which the\nevent happened, or from which the measurement was taken. Host types include\nhardware, virtual machines, Docker containers, and Kubernetes nodes.', + type: 'group', + fields: [ { - name: 'message-id', + name: 'architecture', + level: 'core', type: 'keyword', - description: 'Application message identifier.', + ignore_above: 1024, + description: 'Operating system architecture.', + example: 'x86_64', }, { - name: 'timestamp', + name: 'domain', + level: 'extended', type: 'keyword', - description: 'Message timestamp.', + ignore_above: 1024, + description: + 'Name of the domain of which the host is a member.\n\nFor example, on Windows this could be the host Active Directory domain\nor NetBIOS domain name. For Linux this could be the domain of the host\nLDAP provider.', + example: 'CONTOSO', + default_field: false, }, { - name: 'type', + name: 'geo.city_name', + level: 'core', type: 'keyword', - description: 'Message type name.', + ignore_above: 1024, + description: 'City name.', + example: 'Montreal', }, { - name: 'user-id', + name: 'geo.continent_name', + level: 'core', type: 'keyword', - description: 'Creating user id.', + ignore_above: 1024, + description: 'Name of the continent.', + example: 'North America', }, { - name: 'app-id', + name: 'geo.country_iso_code', + level: 'core', type: 'keyword', - description: 'Creating application id.', - }, - ], - }, - ], - }, - { - key: 'cassandra', - title: 'Cassandra', - description: 'Cassandra v4/3 specific event fields.', - fields: [ - { - name: 'no_request', - type: 'alias', - path: 'cassandra.no_request', - migration: true, - }, - { - name: 'cassandra', - type: 'group', - description: 'Information about the Cassandra request and response.', - fields: [ - { - name: 'no_request', - type: 'boolean', - description: 'Indicates that there is no request because this is a PUSH message.', + ignore_above: 1024, + description: 'Country ISO code.', + example: 'CA', }, { - name: 'request', - type: 'group', - description: 'Cassandra request.', - fields: [ - { - name: 'headers', - type: 'group', - description: 'Cassandra request headers.', - fields: [ - { - name: 'version', - type: 'long', - description: 'The version of the protocol.', - }, - { - name: 'flags', - type: 'keyword', - description: 'Flags applying to this frame.', - }, - { - name: 'stream', - type: 'keyword', - description: - 'A frame has a stream id. If a client sends a request message with the stream id X, it is guaranteed that the stream id of the response to that message will be X.', - }, - { - name: 'op', - type: 'keyword', - description: 'An operation type that distinguishes the actual message.', - }, - { - name: 'length', - type: 'long', - description: - 'A integer representing the length of the body of the frame (a frame is limited to 256MB in length).', - }, - ], - }, - { - name: 'query', - type: 'keyword', - description: 'The CQL query which client send to cassandra.', - }, - ], + name: 'geo.country_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Country name.', + example: 'Canada', }, { - name: 'response', - type: 'group', - description: 'Cassandra response.', - fields: [ - { - name: 'headers', - type: 'group', - description: - "Cassandra response headers, the structure is as same as request's header.", - fields: [ - { - name: 'version', - type: 'long', - description: 'The version of the protocol.', - }, - { - name: 'flags', - type: 'keyword', - description: 'Flags applying to this frame.', - }, - { - name: 'stream', - type: 'keyword', - description: - 'A frame has a stream id. If a client sends a request message with the stream id X, it is guaranteed that the stream id of the response to that message will be X.', - }, - { - name: 'op', - type: 'keyword', - description: 'An operation type that distinguishes the actual message.', - }, - { - name: 'length', - type: 'long', - description: - 'A integer representing the length of the body of the frame (a frame is limited to 256MB in length).', - }, - ], - }, - { - name: 'result', - type: 'group', - description: 'Details about the returned result.', - fields: [ - { - name: 'type', - type: 'keyword', - description: 'Cassandra result type.', - }, - { - name: 'rows', - type: 'group', - description: 'Details about the rows.', - fields: [ - { - name: 'num_rows', - type: 'long', - description: 'Representing the number of rows present in this result.', - }, - { - name: 'meta', - type: 'group', - description: 'Composed of result metadata.', - fields: [ - { - name: 'keyspace', - type: 'keyword', - description: - 'Only present after set Global_tables_spec, the keyspace name.', - }, - { - name: 'table', - type: 'keyword', - description: - 'Only present after set Global_tables_spec, the table name.', - }, - { - name: 'flags', - type: 'keyword', - description: - 'Provides information on the formatting of the remaining information.', - }, - { - name: 'col_count', - type: 'long', - description: - 'Representing the number of columns selected by the query that produced this result.', - }, - { - name: 'pkey_columns', - type: 'long', - description: 'Representing the PK columns index and counts.', - }, - { - name: 'paging_state', - type: 'keyword', - description: - 'The paging_state is a bytes value that should be used in QUERY/EXECUTE to continue paging and retrieve the remainder of the result for this query.', - }, - ], - }, - ], - }, - { - name: 'keyspace', - type: 'keyword', - description: 'Indicating the name of the keyspace that has been set.', - }, - { - name: 'schema_change', - type: 'group', - description: 'The result to a schema_change message.', - fields: [ - { - name: 'change', - type: 'keyword', - description: 'Representing the type of changed involved.', - }, - { - name: 'keyspace', - type: 'keyword', - description: 'This describes which keyspace has changed.', - }, - { - name: 'table', - type: 'keyword', - description: 'This describes which table has changed.', - }, - { - name: 'object', - type: 'keyword', - description: - 'This describes the name of said affected object (either the table, user type, function, or aggregate name).', - }, - { - name: 'target', - type: 'keyword', - description: - 'Target could be "FUNCTION" or "AGGREGATE", multiple arguments.', - }, - { - name: 'name', - type: 'keyword', - description: 'The function/aggregate name.', - }, - { - name: 'args', - type: 'keyword', - description: 'One string for each argument type (as CQL type).', - }, - ], - }, - { - name: 'prepared', - type: 'group', - description: 'The result to a PREPARE message.', - fields: [ - { - name: 'prepared_id', - type: 'keyword', - description: 'Representing the prepared query ID.', - }, - { - name: 'req_meta', - type: 'group', - description: 'This describes the request metadata.', - fields: [ - { - name: 'keyspace', - type: 'keyword', - description: - 'Only present after set Global_tables_spec, the keyspace name.', - }, - { - name: 'table', - type: 'keyword', - description: - 'Only present after set Global_tables_spec, the table name.', - }, - { - name: 'flags', - type: 'keyword', - description: - 'Provides information on the formatting of the remaining information.', - }, - { - name: 'col_count', - type: 'long', - description: - 'Representing the number of columns selected by the query that produced this result.', - }, - { - name: 'pkey_columns', - type: 'long', - description: 'Representing the PK columns index and counts.', - }, - { - name: 'paging_state', - type: 'keyword', - description: - 'The paging_state is a bytes value that should be used in QUERY/EXECUTE to continue paging and retrieve the remainder of the result for this query.', - }, - ], - }, - { - name: 'resp_meta', - type: 'group', - description: 'This describes the metadata for the result set.', - fields: [ - { - name: 'keyspace', - type: 'keyword', - description: - 'Only present after set Global_tables_spec, the keyspace name.', - }, - { - name: 'table', - type: 'keyword', - description: - 'Only present after set Global_tables_spec, the table name.', - }, - { - name: 'flags', - type: 'keyword', - description: - 'Provides information on the formatting of the remaining information.', - }, - { - name: 'col_count', - type: 'long', - description: - 'Representing the number of columns selected by the query that produced this result.', - }, - { - name: 'pkey_columns', - type: 'long', - description: 'Representing the PK columns index and counts.', - }, - { - name: 'paging_state', - type: 'keyword', - description: - 'The paging_state is a bytes value that should be used in QUERY/EXECUTE to continue paging and retrieve the remainder of the result for this query.', - }, - ], - }, - ], - }, - ], - }, - { - name: 'supported', - type: 'object', - object_type: 'keyword', - description: - 'Indicates which startup options are supported by the server. This message comes as a response to an OPTIONS message.', - }, - { - name: 'authentication', - type: 'group', - description: - 'Indicates that the server requires authentication, and which authentication mechanism to use.', - fields: [ - { - name: 'class', - type: 'keyword', - description: 'Indicates the full class name of the IAuthenticator in use', - }, - ], - }, - { - name: 'warnings', - type: 'keyword', - description: 'The text of the warnings, only occur when Warning flag was set.', - }, - { - name: 'event', - type: 'group', - description: - 'Event pushed by the server. A client will only receive events for the types it has REGISTERed to.', - fields: [ - { - name: 'type', - type: 'keyword', - description: 'Representing the event type.', - }, - { - name: 'change', - type: 'keyword', - description: - 'The message corresponding respectively to the type of change followed by the address of the new/removed node.', - }, - { - name: 'host', - type: 'keyword', - description: 'Representing the node ip.', - }, - { - name: 'port', - type: 'long', - description: 'Representing the node port.', - }, - { - name: 'schema_change', - type: 'group', - description: 'The events details related to schema change.', - fields: [ - { - name: 'change', - type: 'keyword', - description: 'Representing the type of changed involved.', - }, - { - name: 'keyspace', - type: 'keyword', - description: 'This describes which keyspace has changed.', - }, - { - name: 'table', - type: 'keyword', - description: 'This describes which table has changed.', - }, - { - name: 'object', - type: 'keyword', - description: - 'This describes the name of said affected object (either the table, user type, function, or aggregate name).', - }, - { - name: 'target', - type: 'keyword', - description: - 'Target could be "FUNCTION" or "AGGREGATE", multiple arguments.', - }, - { - name: 'name', - type: 'keyword', - description: 'The function/aggregate name.', - }, - { - name: 'args', - type: 'keyword', - description: 'One string for each argument type (as CQL type).', - }, - ], - }, - ], - }, - { - name: 'error', - type: 'group', - description: - 'Indicates an error processing a request. The body of the message will be an error code followed by a error message. Then, depending on the exception, more content may follow.', - fields: [ - { - name: 'code', - type: 'long', - description: 'The error code of the Cassandra response.', - }, - { - name: 'msg', - type: 'keyword', - description: 'The error message of the Cassandra response.', - }, - { - name: 'type', - type: 'keyword', - description: 'The error type of the Cassandra response.', - }, - { - name: 'details', - type: 'group', - description: 'The details of the error.', - fields: [ - { - name: 'read_consistency', - type: 'keyword', - description: - 'Representing the consistency level of the query that triggered the exception.', - }, - { - name: 'required', - type: 'long', - description: - 'Representing the number of nodes that should be alive to respect consistency level.', - }, - { - name: 'alive', - type: 'long', - description: - 'Representing the number of replicas that were known to be alive when the request had been processed (since an unavailable exception has been triggered).', - }, - { - name: 'received', - type: 'long', - description: - 'Representing the number of nodes having acknowledged the request.', - }, - { - name: 'blockfor', - type: 'long', - description: - 'Representing the number of replicas whose acknowledgement is required to achieve consistency level.', - }, - { - name: 'write_type', - type: 'keyword', - description: 'Describe the type of the write that timed out.', - }, - { - name: 'data_present', - type: 'boolean', - description: 'It means the replica that was asked for data had responded.', - }, - { - name: 'keyspace', - type: 'keyword', - description: 'The keyspace of the failed function.', - }, - { - name: 'table', - type: 'keyword', - description: 'The keyspace of the failed function.', - }, - { - name: 'stmt_id', - type: 'keyword', - description: 'Representing the unknown ID.', - }, - { - name: 'num_failures', - type: 'keyword', - description: - 'Representing the number of nodes that experience a failure while executing the request.', - }, - { - name: 'function', - type: 'keyword', - description: 'The name of the failed function.', - }, - { - name: 'arg_types', - type: 'keyword', - description: - 'One string for each argument type (as CQL type) of the failed function.', - }, - ], - }, - ], - }, - ], + name: 'geo.location', + level: 'core', + type: 'geo_point', + description: 'Longitude and latitude.', + example: '{ "lon": -73.614830, "lat": 45.505918 }', }, - ], - }, - ], - }, - { - key: 'dhcpv4', - title: 'DHCPv4', - description: 'DHCPv4 event fields', - fields: [ - { - name: 'dhcpv4', - type: 'group', - fields: [ { - name: 'transaction_id', + name: 'geo.name', + level: 'extended', type: 'keyword', + ignore_above: 1024, description: - 'Transaction ID, a random number chosen by the client, used by the client and server to associate messages and responses between a client and a server.', - }, - { - name: 'seconds', - type: 'long', - description: - 'Number of seconds elapsed since client began address acquisition or renewal process.', + 'User-defined description of a location, at the level of granularity\nthey care about.\n\nCould be the name of their data centers, the floor number, if this describes\na local physical entity, city names.\n\nNot typically used in automated geolocation.', + example: 'boston-dc', }, { - name: 'flags', + name: 'geo.region_iso_code', + level: 'core', type: 'keyword', - description: - 'Flags are set by the client to indicate how the DHCP server should its reply -- either unicast or broadcast.', + ignore_above: 1024, + description: 'Region ISO code.', + example: 'CA-QC', }, { - name: 'client_ip', - type: 'ip', - description: 'The current IP address of the client.', + name: 'geo.region_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Region name.', + example: 'Quebec', }, { - name: 'assigned_ip', - type: 'ip', + name: 'hostname', + level: 'core', + type: 'keyword', + ignore_above: 1024, description: - 'The IP address that the DHCP server is assigning to the client. This field is also known as "your" IP address.', + 'Hostname of the host.\n\nIt normally contains what the `hostname` command returns on the host machine.', }, { - name: 'server_ip', - type: 'ip', + name: 'id', + level: 'core', + type: 'keyword', + ignore_above: 1024, description: - 'The IP address of the DHCP server that the client should use for the next step in the bootstrap process.', + 'Unique host id.\n\nAs hostname is not always unique, use values that are meaningful in your environment.\n\nExample: The current usage of `beat.name`.', }, { - name: 'relay_ip', + name: 'ip', + level: 'core', type: 'ip', - description: - 'The relay IP address used by the client to contact the server (i.e. a DHCP relay server).', + description: 'Host ip addresses.', }, { - name: 'client_mac', + name: 'mac', + level: 'core', type: 'keyword', - description: "The client's MAC address (layer two).", + ignore_above: 1024, + description: 'Host mac addresses.', }, { - name: 'server_name', + name: 'name', + level: 'core', type: 'keyword', + ignore_above: 1024, description: - 'The name of the server sending the message. Optional. Used in DHCPOFFER or DHCPACK messages.', + 'Name of the host.\n\nIt can contain what `hostname` returns on Unix systems, the fully qualified\ndomain name, or a name specified by the user. The sender decides which value\nto use.', }, { - name: 'op_code', + name: 'os.family', + level: 'extended', type: 'keyword', - example: 'bootreply', - description: 'The message op code (bootrequest or bootreply).', - }, - { - name: 'hops', - type: 'long', - description: 'The number of hops the DHCP message went through.', + ignore_above: 1024, + description: 'OS family (such as redhat, debian, freebsd, windows).', + example: 'debian', }, { - name: 'hardware_type', + name: 'os.full', + level: 'extended', type: 'keyword', - description: - 'The type of hardware used for the local network (Ethernet, LocalTalk, etc).', - }, - { - name: 'option', - type: 'group', - fields: [ - { - name: 'message_type', - type: 'keyword', - example: 'ack', - description: - 'The specific type of DHCP message being sent (e.g. discover, offer, request, decline, ack, nak, release, inform).', - }, - { - name: 'parameter_request_list', - type: 'keyword', - description: - 'This option is used by a DHCP client to request values for specified configuration parameters.', - }, - { - name: 'requested_ip_address', - type: 'ip', - description: - 'This option is used in a client request (DHCPDISCOVER) to allow the client to request that a particular IP address be assigned.', - }, - { - name: 'server_identifier', - type: 'ip', - description: 'IP address of the individual DHCP server which handled this message.', - }, - { - name: 'broadcast_address', - type: 'ip', - description: - "This option specifies the broadcast address in use on the client's subnet. ", - }, - { - name: 'max_dhcp_message_size', - type: 'long', - description: - 'This option specifies the maximum length DHCP message that the client is willing to accept.', - }, - { - name: 'class_identifier', - type: 'keyword', - description: - "This option is used by DHCP clients to optionally identify the vendor type and configuration of a DHCP client. Vendors may choose to define specific vendor class identifiers to convey particular configuration or other identification information about a client. For example, the identifier may encode the client's hardware configuration. ", - }, - { - name: 'domain_name', - type: 'keyword', - description: - 'This option specifies the domain name that client should use when resolving hostnames via the Domain Name System.', - }, - { - name: 'dns_servers', - type: 'ip', - description: - 'The domain name server option specifies a list of Domain Name System servers available to the client.', - }, - { - name: 'vendor_identifying_options', - type: 'object', - description: - 'A DHCP client may use this option to unambiguously identify the vendor that manufactured the hardware on which the client is running, the software in use, or an industry consortium to which the vendor belongs. This field is described in RFC 3925.', - }, - { - name: 'subnet_mask', - type: 'ip', - description: 'The subnet mask that the client should use on the currnet network.', - }, - { - name: 'utc_time_offset_sec', - type: 'long', - description: - "The time offset field specifies the offset of the client's subnet in seconds from Coordinated Universal Time (UTC). ", - }, - { - name: 'router', - type: 'ip', - description: - "The router option specifies a list of IP addresses for routers on the client's subnet. ", - }, - { - name: 'time_servers', - type: 'ip', - description: - 'The time server option specifies a list of RFC 868 time servers available to the client.', - }, - { - name: 'ntp_servers', - type: 'ip', - description: - 'This option specifies a list of IP addresses indicating NTP servers available to the client.', - }, + ignore_above: 1024, + multi_fields: [ { - name: 'hostname', - type: 'keyword', - description: 'This option specifies the name of the client.', - }, - { - name: 'ip_address_lease_time_sec', - type: 'long', - description: - 'This option is used in a client request (DHCPDISCOVER or DHCPREQUEST) to allow the client to request a lease time for the IP address. In a server reply (DHCPOFFER), a DHCP server uses this option to specify the lease time it is willing to offer.', - }, - { - name: 'message', + name: 'text', type: 'text', - description: - 'This option is used by a DHCP server to provide an error message to a DHCP client in a DHCPNAK message in the event of a failure. A client may use this option in a DHCPDECLINE message to indicate the why the client declined the offered parameters.', - }, - { - name: 'renewal_time_sec', - type: 'long', - description: - 'This option specifies the time interval from address assignment until the client transitions to the RENEWING state.', - }, - { - name: 'rebinding_time_sec', - type: 'long', - description: - 'This option specifies the time interval from address assignment until the client transitions to the REBINDING state.', - }, - { - name: 'boot_file_name', - type: 'keyword', - description: - "This option is used to identify a bootfile when the 'file' field in the DHCP header has been used for DHCP options. ", + norms: false, + default_field: false, }, ], + description: 'Operating system name, including the version or code name.', + example: 'Mac OS Mojave', }, - ], - }, - ], - }, - { - key: 'dns', - title: 'DNS', - description: 'DNS-specific event fields.', - fields: [ - { - name: 'dns', - type: 'group', - fields: [ { - name: 'id', - type: 'long', - description: - 'The DNS packet identifier assigned by the program that generated the query. The identifier is copied to the response.', + name: 'os.kernel', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Operating system kernel version as a raw string.', + example: '4.4.0-112-generic', }, { - name: 'op_code', - description: - 'The DNS operation code that specifies the kind of query in the message. This value is set by the originator of a query and copied into the response.', - example: 'QUERY', + name: 'os.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Operating system name, without the version.', + example: 'Mac OS X', }, { - name: 'flags.authoritative', - type: 'boolean', - description: - 'A DNS flag specifying that the responding server is an authority for the domain name used in the question.', + name: 'os.platform', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Operating system platform (such centos, ubuntu, windows).', + example: 'darwin', }, { - name: 'flags.recursion_available', - type: 'boolean', - description: - 'A DNS flag specifying whether recursive query support is available in the name server.', + name: 'os.version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Operating system version as a raw string.', + example: '10.14.1', }, { - name: 'flags.recursion_desired', - type: 'boolean', + name: 'type', + level: 'core', + type: 'keyword', + ignore_above: 1024, description: - 'A DNS flag specifying that the client directs the server to pursue a query recursively. Recursive query support is optional.', + 'Type of host.\n\nFor Cloud providers this can be the machine type like `t2.medium`. If vm,\nthis could be the container, for example, or other information meaningful\nin your environment.', }, { - name: 'flags.authentic_data', - type: 'boolean', - description: - 'A DNS flag specifying that the recursive server considers the response authentic.', + name: 'uptime', + level: 'extended', + type: 'long', + description: 'Seconds the host has been up.', + example: 1325, }, { - name: 'flags.checking_disabled', - type: 'boolean', + name: 'user.domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, description: - 'A DNS flag specifying that the client disables the server signature validation of the query.', + 'Name of the directory the user is a member of.\n\nFor example, an LDAP or Active Directory domain name.', }, { - name: 'flags.truncated_response', - type: 'boolean', - description: - 'A DNS flag specifying that only the first 512 bytes of the reply were returned.', + name: 'user.email', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'User email address.', }, { - name: 'response_code', - description: 'The DNS status code.', - example: 'NOERROR', + name: 'user.full_name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: "User's full name, if available.", + example: 'Albert Einstein', }, { - name: 'question.name', + name: 'user.group.domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, description: - 'The domain name being queried. If the name field contains non-printable characters (below 32 or above 126), then those characters are represented as escaped base 10 integers (\\DDD). Back slashes and quotes are escaped. Tabs, carriage returns, and line feeds are converted to \\t, \\r, and respectively.', - example: 'www.google.com.', - }, - { - name: 'question.type', - description: 'The type of records being queried.', - example: 'AAAA', + 'Name of the directory the group is a member of.\n\nFor example, an LDAP or Active Directory domain name.', }, { - name: 'question.class', - description: 'The class of of records being queried.', - example: 'IN', + name: 'user.group.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifier for the group on the system/platform.', }, { - name: 'question.etld_plus_one', - description: - 'The effective top-level domain (eTLD) plus one more label. For example, the eTLD+1 for "foo.bar.golang.org." is "golang.org.". The data for determining the eTLD comes from an embedded copy of the data from http://publicsuffix.org.', - example: 'amazon.co.uk.', + name: 'user.group.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the group.', }, { - name: 'answers', - type: 'object', + name: 'user.hash', + level: 'extended', + type: 'keyword', + ignore_above: 1024, description: - 'An array containing a dictionary about each answer section returned by the server.', + 'Unique user hash to correlate information for a user in anonymized\nform.\n\nUseful if `user.id` or `user.name` contain confidential information and cannot\nbe used.', }, { - name: 'answers_count', - type: 'long', - description: 'The number of resource records contained in the `dns.answers` field.', + name: 'user.id', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifiers of the user.', }, { - name: 'answers.name', - description: 'The domain name to which this resource record pertains.', - example: 'example.com.', + name: 'user.name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Short name or login of the user.', + example: 'albert', }, + ], + }, + { + name: 'http', + title: 'HTTP', + group: 2, + description: + 'Fields related to HTTP activity. Use the `url` field set to store\nthe url of the request.', + type: 'group', + fields: [ { - name: 'answers.type', - description: 'The type of data contained in this resource record.', - example: 'MX', + name: 'request.body.bytes', + level: 'extended', + type: 'long', + format: 'bytes', + description: 'Size in bytes of the request body.', + example: 887, }, { - name: 'answers.class', - description: 'The class of DNS data contained in this resource record.', - example: 'IN', + name: 'request.body.content', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'The full HTTP request body.', + example: 'Hello world', }, { - name: 'answers.ttl', - description: - 'The time interval in seconds that this resource record may be cached before it should be discarded. Zero values mean that the data should not be cached.', + name: 'request.bytes', + level: 'extended', type: 'long', + format: 'bytes', + description: 'Total size in bytes of the request (body and headers).', + example: 1437, }, { - name: 'answers.data', + name: 'request.method', + level: 'extended', + type: 'keyword', + ignore_above: 1024, description: - 'The data describing the resource. The meaning of this data depends on the type and class of the resource record.', + 'HTTP request method.\n\nThe field value must be normalized to lowercase for querying. See the documentation\nsection "Implementing ECS".', + example: 'get, post, put', }, { - name: 'authorities', - type: 'object', - description: - 'An array containing a dictionary for each authority section from the answer.', + name: 'request.referrer', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Referrer for this HTTP request.', + example: 'https://blog.example.com/', }, { - name: 'authorities_count', + name: 'response.body.bytes', + level: 'extended', type: 'long', - description: - 'The number of resource records contained in the `dns.authorities` field. The `dns.authorities` field may or may not be included depending on the configuration of Packetbeat.', - }, - { - name: 'authorities.name', - description: 'The domain name to which this resource record pertains.', - example: 'example.com.', + format: 'bytes', + description: 'Size in bytes of the response body.', + example: 887, }, { - name: 'authorities.type', - description: 'The type of data contained in this resource record.', - example: 'NS', + name: 'response.body.content', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'The full HTTP response body.', + example: 'Hello world', }, { - name: 'authorities.class', - description: 'The class of DNS data contained in this resource record.', - example: 'IN', + name: 'response.bytes', + level: 'extended', + type: 'long', + format: 'bytes', + description: 'Total size in bytes of the response (body and headers).', + example: 1437, }, { - name: 'additionals', - type: 'object', - description: - 'An array containing a dictionary for each additional section from the answer.', - }, - { - name: 'additionals_count', + name: 'response.status_code', + level: 'extended', type: 'long', - description: - 'The number of resource records contained in the `dns.additionals` field. The `dns.additionals` field may or may not be included depending on the configuration of Packetbeat.', - }, - { - name: 'additionals.name', - description: 'The domain name to which this resource record pertains.', - example: 'example.com.', - }, - { - name: 'additionals.type', - description: 'The type of data contained in this resource record.', - example: 'NS', - }, - { - name: 'additionals.class', - description: 'The class of DNS data contained in this resource record.', - example: 'IN', + format: 'string', + description: 'HTTP response status code.', + example: 404, }, { - name: 'additionals.ttl', - description: - 'The time interval in seconds that this resource record may be cached before it should be discarded. Zero values mean that the data should not be cached.', - type: 'long', + name: 'version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'HTTP version.', + example: 1.1, }, + ], + }, + { + name: 'interface', + title: 'Interface', + group: 2, + description: + 'The interface fields are used to record ingress and egress interface\ninformation when reported by an observer (e.g. firewall, router, load balancer)\nin the context of the observer handling a network connection. In the case of\na single observer interface (e.g. network sensor on a span port) only the observer.ingress\ninformation should be populated.', + type: 'group', + fields: [ { - name: 'additionals.data', + name: 'alias', + level: 'extended', + type: 'keyword', + ignore_above: 1024, description: - 'The data describing the resource. The meaning of this data depends on the type and class of the resource record.', + 'Interface alias as reported by the system, typically used in firewall\nimplementations for e.g. inside, outside, or dmz logical interface naming.', + example: 'outside', + default_field: false, }, { - name: 'opt.version', - description: 'The EDNS version.', - example: '0', - }, - { - name: 'opt.do', - type: 'boolean', - description: 'If set, the transaction uses DNSSEC.', - }, - { - name: 'opt.ext_rcode', - description: 'Extended response code field.', - example: 'BADVERS', + name: 'id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Interface ID as reported by an observer (typically SNMP interface\nID).', + example: 10, + default_field: false, }, { - name: 'opt.udp_size', - type: 'long', - description: "Requestor's UDP payload size (in bytes).", + name: 'name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Interface name as reported by the system.', + example: 'eth0', + default_field: false, }, ], }, - ], - }, - { - key: 'http', - title: 'HTTP', - description: 'HTTP-specific event fields.', - fields: [ { - name: 'http', + name: 'log', + title: 'Log', + group: 2, + description: + 'Details about the event logging mechanism or logging transport.\n\nThe log.* fields are typically populated with details about the logging mechanism\nused to create and/or transport the event. For example, syslog details belong\nunder `log.syslog.*`.\n\nThe details specific to your event source are typically not logged under `log.*`,\nbut rather in `event.*` or in other ECS fields.', type: 'group', - description: 'Information about the HTTP request and response.', fields: [ { - name: 'request', - description: 'HTTP request', - type: 'group', - fields: [ - { - name: 'headers', - type: 'object', - object_type: 'keyword', - description: - 'A map containing the captured header fields from the request. Which headers to capture is configurable. If headers with the same header name are present in the message, they will be separated by commas.', - }, - { - name: 'params', - type: 'alias', - migration: true, - path: 'url.query', - }, - ], + name: 'level', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'Original log level of the log event.\n\nIf the source of the event provides a log level or textual severity, this\nis the one that goes in `log.level`. If your source does not specify one,\nyou may put your event transport severity here (e.g. Syslog severity).\n\nSome examples are `warn`, `err`, `i`, `informational`.', + example: 'error', }, { - name: 'response', - description: 'HTTP response', - type: 'group', - fields: [ - { - name: 'status_phrase', - description: 'The HTTP status phrase.', - example: 'Not Found', - }, - { - name: 'headers', - type: 'object', - object_type: 'keyword', - description: - 'A map containing the captured header fields from the response. Which headers to capture is configurable. If headers with the same header name are present in the message, they will be separated by commas.', - }, - { - name: 'code', - type: 'alias', - migration: true, - path: 'http.response.status_code', - }, - { - name: 'phrase', - type: 'alias', - migration: true, - path: 'http.response.status_phrase', - }, - ], + name: 'logger', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'The name of the logger inside an application. This is usually the\nname of the class which initialized the logger, or can be a custom name.', + example: 'org.elasticsearch.bootstrap.Bootstrap', }, - ], - }, - ], - }, - { - key: 'icmp', - title: 'ICMP', - description: 'ICMP specific event fields.', - fields: [ - { - name: 'icmp', - type: 'group', - fields: [ { - name: 'version', - description: 'The version of the ICMP protocol.', - possible_values: [4, 6], + name: 'origin.file.line', + level: 'extended', + type: 'integer', + description: + 'The line number of the file containing the source code which originated\nthe log event.', + example: 42, + }, + { + name: 'origin.file.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The name of the file containing the source code which originated\nthe log event. Note that this is not the name of the log file.', + example: 'Bootstrap.java', }, { - name: 'request.message', + name: 'origin.function', + level: 'extended', type: 'keyword', - description: 'A human readable form of the request.', + ignore_above: 1024, + description: 'The name of the function or method which originated the log event.', + example: 'init', }, { - name: 'request.type', - type: 'long', - description: 'The request type.', + name: 'original', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'This is the original log message and contains the full log message\nbefore splitting it up in multiple parts.\n\nIn contrast to the `message` field which can contain an extracted part of\nthe log message, this field contains the original, full log message. It can\nhave already some modifications applied like encoding or new lines removed\nto clean up the log message.\n\nThis field is not indexed and doc_values are disabled so it cannot be queried\nbut the value can be retrieved from `_source`.', + example: 'Sep 19 08:26:10 localhost My log', + }, + { + name: 'syslog', + level: 'extended', + type: 'object', + object_type: 'keyword', + description: + 'The Syslog metadata of the event, if the event was transmitted\nvia Syslog. Please see RFCs 5424 or 3164.', }, { - name: 'request.code', + name: 'syslog.facility.code', + level: 'extended', type: 'long', - description: 'The request code.', + format: 'string', + description: + 'The Syslog numeric facility of the log event, if available.\n\nAccording to RFCs 5424 and 3164, this value should be an integer between 0\nand 23.', + example: 23, }, { - name: 'response.message', + name: 'syslog.facility.name', + level: 'extended', type: 'keyword', - description: 'A human readable form of the response.', + ignore_above: 1024, + description: 'The Syslog text-based facility of the log event, if available.', + example: 'local7', }, { - name: 'response.type', + name: 'syslog.priority', + level: 'extended', type: 'long', - description: 'The response type.', + format: 'string', + description: + 'Syslog numeric priority of the event, if available.\n\nAccording to RFCs 5424 and 3164, the priority is 8 * facility + severity.\nThis number is therefore expected to contain a value between 0 and 191.', + example: 135, }, { - name: 'response.code', + name: 'syslog.severity.code', + level: 'extended', type: 'long', - description: 'The response code.', + description: + 'The Syslog numeric severity of the log event, if available.\n\nIf the event source publishing via Syslog provides a different numeric severity\nvalue (e.g. firewall, IDS), your source numeric severity should go to `event.severity`.\nIf the event source does not specify a distinct severity, you can optionally\ncopy the Syslog severity to `event.severity`.', + example: 3, + }, + { + name: 'syslog.severity.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The Syslog numeric severity of the log event, if available.\n\nIf the event source publishing via Syslog provides a different severity value\n(e.g. firewall, IDS), your source text severity should go to `log.level`.\nIf the event source does not specify a distinct severity, you can optionally\ncopy the Syslog severity to `log.level`.', + example: 'Error', }, ], }, - ], - }, - { - key: 'memcache', - title: 'Memcache', - description: 'Memcached-specific event fields', - fields: [ { - name: 'memcache', + name: 'network', + title: 'Network', + group: 2, + description: + 'The network is defined as the communication path over which a host\nor network event happens.\n\nThe network.* fields should be populated with details about the network activity\nassociated with an event.', type: 'group', fields: [ { - name: 'protocol_type', + name: 'application', + level: 'extended', type: 'keyword', + ignore_above: 1024, description: - 'The memcache protocol implementation. The value can be "binary" for binary-based, "text" for text-based, or "unknown" for an unknown memcache protocol type.', + 'A name given to an application level protocol. This can be arbitrarily\nassigned for things like microservices, but also apply to things like skype,\nicq, facebook, twitter. This would be used in situations where the vendor\nor service can be decoded such as from the source/dest IP owners, ports, or\nwire format.\n\nThe field value must be normalized to lowercase for querying. See the documentation\nsection "Implementing ECS".', + example: 'aim', }, { - name: 'request.line', - type: 'keyword', - description: 'The raw command line for unknown commands ONLY.', + name: 'bytes', + level: 'core', + type: 'long', + format: 'bytes', + description: + 'Total bytes transferred in both directions.\n\nIf `source.bytes` and `destination.bytes` are known, `network.bytes` is their\nsum.', + example: 368, }, { - name: 'request.command', + name: 'community_id', + level: 'extended', type: 'keyword', + ignore_above: 1024, description: - 'The memcache command being requested in the memcache text protocol. For example "set" or "get". The binary protocol opcodes are translated into memcache text protocol commands.', + 'A hash of source and destination IPs and ports, as well as the\nprotocol used in a communication. This is a tool-agnostic standard to identify\nflows.\n\nLearn more at https://github.com/corelight/community-id-spec.', + example: '1:hO+sN4H+MG5MY/8hIrXPqc4ZQz0=', }, { - name: 'response.command', + name: 'direction', + level: 'core', type: 'keyword', + ignore_above: 1024, description: - 'Either the text based protocol response message type or the name of the originating request if binary protocol is used.', + "Direction of the network traffic.\nRecommended values are:\n * inbound\n * outbound\n * internal\n * external\n * unknown\n\nWhen mapping events from a host-based monitoring context, populate this field from the host's point of view.\nWhen mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of your network perimeter.", + example: 'inbound', }, { - name: 'request.type', - type: 'keyword', - description: - 'The memcache command classification. This value can be "UNKNOWN", "Load", "Store", "Delete", "Counter", "Info", "SlabCtrl", "LRUCrawler", "Stats", "Success", "Fail", or "Auth".', + name: 'forwarded_ip', + level: 'core', + type: 'ip', + description: 'Host IP address when the source IP address is the proxy.', + example: '192.1.1.2', }, { - name: 'response.type', + name: 'iana_number', + level: 'extended', type: 'keyword', + ignore_above: 1024, description: - 'The memcache command classification. This value can be "UNKNOWN", "Load", "Store", "Delete", "Counter", "Info", "SlabCtrl", "LRUCrawler", "Stats", "Success", "Fail", or "Auth". The text based protocol will employ any of these, whereas the binary based protocol will mirror the request commands only (see `memcache.response.status` for binary protocol).', + 'IANA Protocol Number (https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml).\nStandardized list of protocols. This aligns well with NetFlow and sFlow related\nlogs which use the IANA Protocol Number.', + example: 6, }, { - name: 'response.error_msg', - type: 'keyword', + name: 'inner', + level: 'extended', + type: 'object', + object_type: 'keyword', description: - 'The optional error message in the memcache response (text based protocol only).', + 'Network.inner fields are added in addition to network.vlan fields\nto describe the innermost VLAN when q-in-q VLAN tagging is present. Allowed\nfields include vlan.id and vlan.name. Inner vlan fields are typically used\nwhen sending traffic with multiple 802.1q encapsulations to a network sensor\n(e.g. Zeek, Wireshark.)', + default_field: false, }, { - name: 'request.opcode', + name: 'inner.vlan.id', + level: 'extended', type: 'keyword', - description: 'The binary protocol message opcode name.', + ignore_above: 1024, + description: 'VLAN ID as reported by the observer.', + example: 10, + default_field: false, }, { - name: 'response.opcode', + name: 'inner.vlan.name', + level: 'extended', type: 'keyword', - description: 'The binary protocol message opcode name.', + ignore_above: 1024, + description: 'Optional VLAN name as reported by the observer.', + example: 'outside', + default_field: false, }, { - name: 'request.opcode_value', - type: 'long', - description: 'The binary protocol message opcode value.', - }, - { - name: 'response.opcode_value', - type: 'long', - description: 'The binary protocol message opcode value.', + name: 'name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Name given by operators to sections of their network.', + example: 'Guest Wifi', }, { - name: 'request.opaque', + name: 'packets', + level: 'core', type: 'long', description: - 'The binary protocol opaque header value used for correlating request with response messages.', + 'Total packets transferred in both directions.\n\nIf `source.packets` and `destination.packets` are known, `network.packets`\nis their sum.', + example: 24, }, { - name: 'response.opaque', - type: 'long', + name: 'protocol', + level: 'core', + type: 'keyword', + ignore_above: 1024, description: - 'The binary protocol opaque header value used for correlating request with response messages.', - }, - { - name: 'request.vbucket', - type: 'long', - description: 'The vbucket index sent in the binary message.', + 'L7 Network protocol name. ex. http, lumberjack, transport protocol.\n\nThe field value must be normalized to lowercase for querying. See the documentation\nsection "Implementing ECS".', + example: 'http', }, { - name: 'response.status', + name: 'transport', + level: 'core', type: 'keyword', + ignore_above: 1024, description: - 'The textual representation of the response error code (binary protocol only).', + 'Same as network.iana_number, but instead using the Keyword name\nof the transport layer (udp, tcp, ipv6-icmp, etc.)\n\nThe field value must be normalized to lowercase for querying. See the documentation\nsection "Implementing ECS".', + example: 'tcp', }, { - name: 'response.status_code', - type: 'long', - description: 'The status code value returned in the response (binary protocol only).', + name: 'type', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'In the OSI Model this would be the Network Layer. ipv4, ipv6,\nipsec, pim, etc\n\nThe field value must be normalized to lowercase for querying. See the documentation\nsection "Implementing ECS".', + example: 'ipv4', }, { - name: 'request.keys', - type: 'array', - description: 'The list of keys sent in the store or load commands.', + name: 'vlan.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'VLAN ID as reported by the observer.', + example: 10, + default_field: false, }, { - name: 'response.keys', - type: 'array', - description: 'The list of keys returned for the load command (if present).', + name: 'vlan.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Optional VLAN name as reported by the observer.', + example: 'outside', + default_field: false, }, + ], + }, + { + name: 'observer', + title: 'Observer', + group: 2, + description: + 'An observer is defined as a special network, security, or application\ndevice used to detect, observe, or create network, security, or application-related\nevents and metrics.\n\nThis could be a custom hardware appliance or a server that has been configured\nto run special network, security, or application software. Examples include\nfirewalls, web proxies, intrusion detection/prevention systems, network monitoring\nsensors, web application firewalls, data loss prevention systems, and APM servers.\nThe observer.* fields shall be populated with details of the system, if any,\nthat detects, observes and/or creates a network, security, or application event\nor metric. Message queues and ETL components used in processing events or metrics\nare not considered observers in ECS.', + type: 'group', + fields: [ { - name: 'request.count_values', - type: 'long', + name: 'egress', + level: 'extended', + type: 'object', + object_type: 'keyword', description: - 'The number of values found in the memcache request message. If the command does not send any data, this field is missing.', + 'Observer.egress holds information like interface number and name,\nvlan, and zone information to classify egress traffic. Single armed monitoring\nsuch as a network sensor on a span port should only use observer.ingress\nto categorize traffic.', + default_field: false, }, { - name: 'response.count_values', - type: 'long', + name: 'egress.interface.alias', + level: 'extended', + type: 'keyword', + ignore_above: 1024, description: - 'The number of values found in the memcache response message. If the command does not send any data, this field is missing.', + 'Interface alias as reported by the system, typically used in firewall\nimplementations for e.g. inside, outside, or dmz logical interface naming.', + example: 'outside', + default_field: false, }, { - name: 'request.values', - type: 'array', - description: 'The list of base64 encoded values sent with the request (if present).', + name: 'egress.interface.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Interface ID as reported by an observer (typically SNMP interface\nID).', + example: 10, + default_field: false, }, { - name: 'response.values', - type: 'array', - description: 'The list of base64 encoded values sent with the response (if present).', + name: 'egress.interface.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Interface name as reported by the system.', + example: 'eth0', + default_field: false, }, { - name: 'request.bytes', - type: 'long', - format: 'bytes', - description: 'The byte count of the values being transferred.', + name: 'egress.vlan.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'VLAN ID as reported by the observer.', + example: 10, + default_field: false, }, { - name: 'response.bytes', - type: 'long', - format: 'bytes', - description: 'The byte count of the values being transferred.', + name: 'egress.vlan.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Optional VLAN name as reported by the observer.', + example: 'outside', + default_field: false, }, { - name: 'request.delta', - type: 'long', - description: 'The counter increment/decrement delta value.', + name: 'egress.zone', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Network zone of outbound traffic as reported by the observer to\ncategorize the destination area of egress traffic, e.g. Internal, External,\nDMZ, HR, Legal, etc.', + example: 'Public_Internet', + default_field: false, }, { - name: 'request.initial', - type: 'long', - description: - 'The counter increment/decrement initial value parameter (binary protocol only).', + name: 'geo.city_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'City name.', + example: 'Montreal', }, { - name: 'request.verbosity', - type: 'long', - description: 'The value of the memcache "verbosity" command.', + name: 'geo.continent_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the continent.', + example: 'North America', }, { - name: 'request.raw_args', + name: 'geo.country_iso_code', + level: 'core', type: 'keyword', - description: - 'The text protocol raw arguments for the "stats ..." and "lru crawl ..." commands.', + ignore_above: 1024, + description: 'Country ISO code.', + example: 'CA', }, { - name: 'request.source_class', - type: 'long', - description: "The source class id in 'slab reassign' command. ", + name: 'geo.country_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Country name.', + example: 'Canada', }, { - name: 'request.dest_class', - type: 'long', - description: "The destination class id in 'slab reassign' command. ", + name: 'geo.location', + level: 'core', + type: 'geo_point', + description: 'Longitude and latitude.', + example: '{ "lon": -73.614830, "lat": 45.505918 }', }, { - name: 'request.automove', + name: 'geo.name', + level: 'extended', type: 'keyword', + ignore_above: 1024, description: - 'The automove mode in the \'slab automove\' command expressed as a string. This value can be "standby"(=0), "slow"(=1), "aggressive"(=2), or the raw value if the value is unknown.', + 'User-defined description of a location, at the level of granularity\nthey care about.\n\nCould be the name of their data centers, the floor number, if this describes\na local physical entity, city names.\n\nNot typically used in automated geolocation.', + example: 'boston-dc', }, { - name: 'request.flags', - type: 'long', - description: 'The memcache command flags sent in the request (if present).', + name: 'geo.region_iso_code', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Region ISO code.', + example: 'CA-QC', }, { - name: 'response.flags', - type: 'long', - description: 'The memcache message flags sent in the response (if present).', + name: 'geo.region_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Region name.', + example: 'Quebec', }, { - name: 'request.exptime', - type: 'long', + name: 'hostname', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Hostname of the observer.', + }, + { + name: 'ingress', + level: 'extended', + type: 'object', + object_type: 'keyword', description: - 'The data expiry time in seconds sent with the memcache command (if present). If the value is <30 days, the expiry time is relative to "now", or else it is an absolute Unix time in seconds (32-bit).', + 'Observer.ingress holds information like interface number and name,\nvlan, and zone information to classify ingress traffic. Single armed monitoring\nsuch as a network sensor on a span port should only use observer.ingress\nto categorize traffic.', + default_field: false, }, { - name: 'request.sleep_us', - type: 'long', - description: "The sleep setting in microseconds for the 'lru_crawler sleep' command. ", + name: 'ingress.interface.alias', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Interface alias as reported by the system, typically used in firewall\nimplementations for e.g. inside, outside, or dmz logical interface naming.', + example: 'outside', + default_field: false, }, { - name: 'response.value', - type: 'long', - description: 'The counter value returned by a counter operation.', + name: 'ingress.interface.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Interface ID as reported by an observer (typically SNMP interface\nID).', + example: 10, + default_field: false, }, { - name: 'request.noreply', - type: 'boolean', - description: - 'Set to true if noreply was set in the request. The `memcache.response` field will be missing.', + name: 'ingress.interface.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Interface name as reported by the system.', + example: 'eth0', + default_field: false, }, { - name: 'request.quiet', - type: 'boolean', + name: 'ingress.vlan.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'VLAN ID as reported by the observer.', + example: 10, + default_field: false, + }, + { + name: 'ingress.vlan.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Optional VLAN name as reported by the observer.', + example: 'outside', + default_field: false, + }, + { + name: 'ingress.zone', + level: 'extended', + type: 'keyword', + ignore_above: 1024, description: - 'Set to true if the binary protocol message is to be treated as a quiet message.', + 'Network zone of incoming traffic as reported by the observer to\ncategorize the source area of ingress traffic. e.g. internal, External, DMZ,\nHR, Legal, etc.', + example: 'DMZ', + default_field: false, }, { - name: 'request.cas_unique', - type: 'long', - description: 'The CAS (compare-and-swap) identifier if present.', + name: 'ip', + level: 'core', + type: 'ip', + description: 'IP addresses of the observer.', }, { - name: 'response.cas_unique', - type: 'long', + name: 'mac', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'MAC addresses of the observer', + }, + { + name: 'name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, description: - 'The CAS (compare-and-swap) identifier to be used with CAS-based updates (if present).', + 'Custom name of the observer.\n\nThis is a name that can be given to an observer. This can be helpful for example\nif multiple firewalls of the same model are used in an organization.\n\nIf no custom name is needed, the field can be left empty.', + example: '1_proxySG', + }, + { + name: 'os.family', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'OS family (such as redhat, debian, freebsd, windows).', + example: 'debian', + }, + { + name: 'os.full', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Operating system name, including the version or code name.', + example: 'Mac OS Mojave', + }, + { + name: 'os.kernel', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Operating system kernel version as a raw string.', + example: '4.4.0-112-generic', + }, + { + name: 'os.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Operating system name, without the version.', + example: 'Mac OS X', + }, + { + name: 'os.platform', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Operating system platform (such centos, ubuntu, windows).', + example: 'darwin', + }, + { + name: 'os.version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Operating system version as a raw string.', + example: '10.14.1', + }, + { + name: 'product', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'The product name of the observer.', + example: 's200', + }, + { + name: 'serial_number', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Observer serial number.', }, { - name: 'response.stats', - type: 'array', + name: 'type', + level: 'core', + type: 'keyword', + ignore_above: 1024, description: - 'The list of statistic values returned. Each entry is a dictionary with the fields "name" and "value".', + 'The type of the observer the data is coming from.\n\nThere is no predefined list of observer types. Some examples are `forwarder`,\n`firewall`, `ids`, `ips`, `proxy`, `poller`, `sensor`, `APM server`.', + example: 'firewall', + }, + { + name: 'vendor', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Vendor name of the observer.', + example: 'Symantec', }, { - name: 'response.version', + name: 'version', + level: 'core', type: 'keyword', - description: 'The returned memcache version string.', + ignore_above: 1024, + description: 'Observer version.', }, ], }, - ], - }, - { - key: 'mongodb', - title: 'MongoDb', - description: - 'MongoDB-specific event fields. These fields mirror closely the fields for the MongoDB wire protocol. The higher level fields (for example, `query` and `resource`) apply to MongoDB events as well.', - fields: [ { - name: 'mongodb', + name: 'organization', + title: 'Organization', + group: 2, + description: + 'The organization fields enrich data with information about the company\nor entity the data is associated with.\n\nThese fields help you arrange or filter data stored in an index by one or multiple\norganizations.', type: 'group', fields: [ { - name: 'error', - description: - 'If the MongoDB request has resulted in an error, this field contains the error message returned by the server.', + name: 'id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifier for the organization.', }, { - name: 'fullCollectionName', - description: - 'The full collection name. The full collection name is the concatenation of the database name with the collection name, using a dot (.) for the concatenation. For example, for the database foo and the collection bar, the full collection name is foo.bar.', + name: 'name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Organization name.', + }, + ], + }, + { + name: 'os', + title: 'Operating System', + group: 2, + description: 'The OS fields contain information about the operating system.', + type: 'group', + fields: [ + { + name: 'family', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'OS family (such as redhat, debian, freebsd, windows).', + example: 'debian', }, { - name: 'numberToSkip', - type: 'long', + name: 'full', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Operating system name, including the version or code name.', + example: 'Mac OS Mojave', + }, + { + name: 'kernel', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Operating system kernel version as a raw string.', + example: '4.4.0-112-generic', + }, + { + name: 'name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Operating system name, without the version.', + example: 'Mac OS X', + }, + { + name: 'platform', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Operating system platform (such centos, ubuntu, windows).', + example: 'darwin', + }, + { + name: 'version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Operating system version as a raw string.', + example: '10.14.1', + }, + ], + }, + { + name: 'package', + title: 'Package', + group: 2, + description: + 'These fields contain information about an installed software package.\nIt contains general information about a package, such as name, version or size.\nIt also contains installation details, such as time or location.', + type: 'group', + fields: [ + { + name: 'architecture', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Package architecture.', + example: 'x86_64', + }, + { + name: 'build_version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, description: - 'Sets the number of documents to omit - starting from the first document in the resulting dataset - when returning the result of the query.', + 'Additional information about the build version of the installed\npackage.\n\nFor example use the commit SHA of a non-released package.', + example: '36f4f7e89dd61b0988b12ee000b98966867710cd', + default_field: false, }, { - name: 'numberToReturn', - type: 'long', - description: 'The requested maximum number of documents to be returned.', + name: 'checksum', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Checksum of the installed package for verification.', + example: '68b329da9893e34099c7d8ad5cb9c940', }, { - name: 'numberReturned', - type: 'long', - description: 'The number of documents in the reply.', + name: 'description', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Description of the package.', + example: + 'Open source programming language to build simple/reliable/efficient\nsoftware.', }, { - name: 'startingFrom', - description: 'Where in the cursor this reply is starting.', + name: 'install_scope', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Indicating how the package was installed, e.g. user-local, global.', + example: 'global', }, { - name: 'query', + name: 'installed', + level: 'extended', + type: 'date', + description: 'Time when package was installed.', + }, + { + name: 'license', + level: 'extended', + type: 'keyword', + ignore_above: 1024, description: - 'A JSON document that represents the query. The query will contain one or more elements, all of which must match for a document to be included in the result set. Possible elements include $query, $orderby, $hint, $explain, and $snapshot.', + 'License under which the package was released.\n\nUse a short name, e.g. the license identifier from SPDX License List where\npossible (https://spdx.org/licenses/).', + example: 'Apache License 2.0', + }, + { + name: 'name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Package name', + example: 'go', + }, + { + name: 'path', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Path where the package is installed.', + example: '/usr/local/Cellar/go/1.12.9/', }, { - name: 'returnFieldsSelector', + name: 'reference', + level: 'extended', + type: 'keyword', + ignore_above: 1024, description: - 'A JSON document that limits the fields in the returned documents. The returnFieldsSelector contains one or more elements, each of which is the name of a field that should be returned, and the integer value 1.', + 'Home page or reference URL of the software in this package, if\navailable.', + example: 'https://golang.org', + default_field: false, + }, + { + name: 'size', + level: 'extended', + type: 'long', + format: 'string', + description: 'Package size in bytes.', + example: 62231, }, { - name: 'selector', + name: 'type', + level: 'extended', + type: 'keyword', + ignore_above: 1024, description: - 'A BSON document that specifies the query for selecting the document to update or delete.', + 'Type of package.\n\nThis should contain the package file type, rather than the package manager\nname. Examples: rpm, dpkg, brew, npm, gem, nupkg, jar.', + example: 'rpm', + default_field: false, + }, + { + name: 'version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Package version', + example: '1.12.9', + }, + ], + }, + { + name: 'pe', + title: 'PE Header', + group: 2, + description: 'These fields contain Windows Portable Executable (PE) metadata.', + type: 'group', + fields: [ + { + name: 'company', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal company name of the file, provided at compile-time.', + example: 'Microsoft Corporation', + default_field: false, + }, + { + name: 'description', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal description of the file, provided at compile-time.', + example: 'Paint', + default_field: false, + }, + { + name: 'file_version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal version of the file, provided at compile-time.', + example: '6.3.9600.17415', + default_field: false, + }, + { + name: 'original_file_name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal name of the file, provided at compile-time.', + example: 'MSPAINT.EXE', + default_field: false, + }, + { + name: 'product', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal product name of the file, provided at compile-time.', + example: 'Microsoft® Windows® Operating System', + default_field: false, + }, + ], + }, + { + name: 'process', + title: 'Process', + group: 2, + description: + 'These fields contain information about a process.\n\nThese fields can help you correlate metrics information with a process id/name\nfrom a log message. The `process.pid` often stays in the metric itself and\nis copied to the global field for correlation.', + type: 'group', + fields: [ + { + name: 'args', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Array of process arguments, starting with the absolute path to\nthe executable.\n\nMay be filtered to protect sensitive information.', + example: ['/usr/bin/ssh', '-l', 'user', '10.0.0.16'], + }, + { + name: 'args_count', + level: 'extended', + type: 'long', + description: + 'Length of the process.args array.\n\nThis field can be useful for querying or performing bucket analysis on how\nmany arguments were provided to start a process. More arguments may be an\nindication of suspicious activity.', + example: 4, + default_field: false, + }, + { + name: 'code_signature.exists', + level: 'core', + type: 'boolean', + description: 'Boolean to capture if a signature is present.', + example: 'true', + default_field: false, + }, + { + name: 'code_signature.status', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Additional information about the certificate status.\n\nThis is useful for logging cryptographic errors with the certificate validity\nor trust status. Leave unpopulated if the validity or trust of the certificate\nwas unchecked.', + example: 'ERROR_UNTRUSTED_ROOT', + default_field: false, + }, + { + name: 'code_signature.subject_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Subject name of the code signer', + example: 'Microsoft Corporation', + default_field: false, + }, + { + name: 'code_signature.trusted', + level: 'extended', + type: 'boolean', + description: + 'Stores the trust status of the certificate chain.\n\nValidating the trust of the certificate chain may be complicated, and this\nfield should only be populated by tools that actively check the status.', + example: 'true', + default_field: false, + }, + { + name: 'code_signature.valid', + level: 'extended', + type: 'boolean', + description: + 'Boolean to capture if the digital signature is verified against\nthe binary content.\n\nLeave unpopulated if a certificate was unchecked.', + example: 'true', + default_field: false, + }, + { + name: 'command_line', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + }, + ], + description: + 'Full command line that started the process, including the absolute\npath to the executable, and all arguments.\n\nSome arguments may be filtered to protect sensitive information.', + example: '/usr/bin/ssh -l user 10.0.0.16', + default_field: false, + }, + { + name: 'entity_id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Unique identifier for the process.\n\nThe implementation of this is specified by the data source, but some examples\nof what could be used here are a process-generated UUID, Sysmon Process GUIDs,\nor a hash of some uniquely identifying components of a process.\n\nConstructing a globally unique identifier is a common practice to mitigate\nPID reuse as well as to identify a specific process over time, across multiple\nmonitored hosts.', + example: 'c2c455d9f99375d', + default_field: false, + }, + { + name: 'executable', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Absolute path to the process executable.', + example: '/usr/bin/ssh', + }, + { + name: 'exit_code', + level: 'extended', + type: 'long', + description: + 'The exit code of the process, if this is a termination event.\n\nThe field should be absent if there is no exit code for the event (e.g. process\nstart).', + example: 137, + default_field: false, + }, + { + name: 'hash.md5', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'MD5 hash.', + }, + { + name: 'hash.sha1', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'SHA1 hash.', + }, + { + name: 'hash.sha256', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'SHA256 hash.', + }, + { + name: 'hash.sha512', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'SHA512 hash.', + }, + { + name: 'name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Process name.\n\nSometimes called program name or similar.', + example: 'ssh', + }, + { + name: 'parent.args', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Array of process arguments.\n\nMay be filtered to protect sensitive information.', + example: ['ssh', '-l', 'user', '10.0.0.16'], + default_field: false, + }, + { + name: 'parent.args_count', + level: 'extended', + type: 'long', + description: + 'Length of the process.args array.\n\nThis field can be useful for querying or performing bucket analysis on how\nmany arguments were provided to start a process. More arguments may be an\nindication of suspicious activity.', + example: 4, + default_field: false, + }, + { + name: 'parent.code_signature.exists', + level: 'core', + type: 'boolean', + description: 'Boolean to capture if a signature is present.', + example: 'true', + default_field: false, + }, + { + name: 'parent.code_signature.status', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Additional information about the certificate status.\n\nThis is useful for logging cryptographic errors with the certificate validity\nor trust status. Leave unpopulated if the validity or trust of the certificate\nwas unchecked.', + example: 'ERROR_UNTRUSTED_ROOT', + default_field: false, + }, + { + name: 'parent.code_signature.subject_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Subject name of the code signer', + example: 'Microsoft Corporation', + default_field: false, + }, + { + name: 'parent.code_signature.trusted', + level: 'extended', + type: 'boolean', + description: + 'Stores the trust status of the certificate chain.\n\nValidating the trust of the certificate chain may be complicated, and this\nfield should only be populated by tools that actively check the status.', + example: 'true', + default_field: false, + }, + { + name: 'parent.code_signature.valid', + level: 'extended', + type: 'boolean', + description: + 'Boolean to capture if the digital signature is verified against\nthe binary content.\n\nLeave unpopulated if a certificate was unchecked.', + example: 'true', + default_field: false, + }, + { + name: 'parent.command_line', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + }, + ], + description: + 'Full command line that started the process, including the absolute\npath to the executable, and all arguments.\n\nSome arguments may be filtered to protect sensitive information.', + example: '/usr/bin/ssh -l user 10.0.0.16', + default_field: false, + }, + { + name: 'parent.entity_id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Unique identifier for the process.\n\nThe implementation of this is specified by the data source, but some examples\nof what could be used here are a process-generated UUID, Sysmon Process GUIDs,\nor a hash of some uniquely identifying components of a process.\n\nConstructing a globally unique identifier is a common practice to mitigate\nPID reuse as well as to identify a specific process over time, across multiple\nmonitored hosts.', + example: 'c2c455d9f99375d', + default_field: false, + }, + { + name: 'parent.executable', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + }, + ], + description: 'Absolute path to the process executable.', + example: '/usr/bin/ssh', + default_field: false, + }, + { + name: 'parent.exit_code', + level: 'extended', + type: 'long', + description: + 'The exit code of the process, if this is a termination event.\n\nThe field should be absent if there is no exit code for the event (e.g. process\nstart).', + example: 137, + default_field: false, + }, + { + name: 'parent.hash.md5', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'MD5 hash.', + default_field: false, + }, + { + name: 'parent.hash.sha1', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'SHA1 hash.', + default_field: false, + }, + { + name: 'parent.hash.sha256', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'SHA256 hash.', + default_field: false, + }, + { + name: 'parent.hash.sha512', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'SHA512 hash.', + default_field: false, + }, + { + name: 'parent.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + }, + ], + description: 'Process name.\n\nSometimes called program name or similar.', + example: 'ssh', + default_field: false, + }, + { + name: 'parent.pgid', + level: 'extended', + type: 'long', + format: 'string', + description: 'Identifier of the group of processes the process belongs to.', + default_field: false, + }, + { + name: 'parent.pid', + level: 'core', + type: 'long', + format: 'string', + description: 'Process id.', + example: 4242, + default_field: false, + }, + { + name: 'parent.ppid', + level: 'extended', + type: 'long', + format: 'string', + description: "Parent process' pid.", + example: 4241, + default_field: false, + }, + { + name: 'parent.start', + level: 'extended', + type: 'date', + description: 'The time the process started.', + example: '2016-05-23T08:05:34.853Z', + default_field: false, + }, + { + name: 'parent.thread.id', + level: 'extended', + type: 'long', + format: 'string', + description: 'Thread ID.', + example: 4242, + default_field: false, + }, + { + name: 'parent.thread.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Thread name.', + example: 'thread-0', + default_field: false, + }, + { + name: 'parent.title', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + }, + ], + description: + 'Process title.\n\nThe proctitle, some times the same as process name. Can also be different:\nfor example a browser setting its title to the web page currently opened.', + default_field: false, + }, + { + name: 'parent.uptime', + level: 'extended', + type: 'long', + description: 'Seconds the process has been up.', + example: 1325, + default_field: false, + }, + { + name: 'parent.working_directory', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + }, + ], + description: 'The working directory of the process.', + example: '/home/alice', + default_field: false, + }, + { + name: 'pe.company', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal company name of the file, provided at compile-time.', + example: 'Microsoft Corporation', + default_field: false, + }, + { + name: 'pe.description', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal description of the file, provided at compile-time.', + example: 'Paint', + default_field: false, + }, + { + name: 'pe.file_version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal version of the file, provided at compile-time.', + example: '6.3.9600.17415', + default_field: false, + }, + { + name: 'pe.original_file_name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal name of the file, provided at compile-time.', + example: 'MSPAINT.EXE', + default_field: false, + }, + { + name: 'pe.product', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal product name of the file, provided at compile-time.', + example: 'Microsoft® Windows® Operating System', + default_field: false, + }, + { + name: 'pgid', + level: 'extended', + type: 'long', + format: 'string', + description: 'Identifier of the group of processes the process belongs to.', + }, + { + name: 'pid', + level: 'core', + type: 'long', + format: 'string', + description: 'Process id.', + example: 4242, + }, + { + name: 'ppid', + level: 'extended', + type: 'long', + format: 'string', + description: "Parent process' pid.", + example: 4241, + }, + { + name: 'start', + level: 'extended', + type: 'date', + description: 'The time the process started.', + example: '2016-05-23T08:05:34.853Z', + }, + { + name: 'thread.id', + level: 'extended', + type: 'long', + format: 'string', + description: 'Thread ID.', + example: 4242, + }, + { + name: 'thread.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Thread name.', + example: 'thread-0', + }, + { + name: 'title', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: + 'Process title.\n\nThe proctitle, some times the same as process name. Can also be different:\nfor example a browser setting its title to the web page currently opened.', + }, + { + name: 'uptime', + level: 'extended', + type: 'long', + description: 'Seconds the process has been up.', + example: 1325, + }, + { + name: 'working_directory', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'The working directory of the process.', + example: '/home/alice', + }, + ], + }, + { + name: 'registry', + title: 'Registry', + group: 2, + description: 'Fields related to Windows Registry operations.', + type: 'group', + fields: [ + { + name: 'data.bytes', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Original bytes written with base64 encoding.\n\nFor Windows registry operations, such as SetValueEx and RegQueryValueEx, this\ncorresponds to the data pointed by `lp_data`. This is optional but provides\nbetter recoverability and should be populated for REG_BINARY encoded values.', + example: 'ZQBuAC0AVQBTAAAAZQBuAAAAAAA=', + default_field: false, + }, + { + name: 'data.strings', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'Content when writing string types.\n\nPopulated as an array when writing string data to the registry. For single\nstring registry types (REG_SZ, REG_EXPAND_SZ), this should be an array with\none string. For sequences of string with REG_MULTI_SZ, this array will be\nvariable length. For numeric data, such as REG_DWORD and REG_QWORD, this should\nbe populated with the decimal representation (e.g `"1"`).', + example: '["C:\\rta\\red_ttp\\bin\\myapp.exe"]', + default_field: false, + }, + { + name: 'data.type', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Standard registry type for encoding contents', + example: 'REG_SZ', + default_field: false, + }, + { + name: 'hive', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Abbreviated name for the hive.', + example: 'HKLM', + default_field: false, + }, + { + name: 'key', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Hive-relative path of keys.', + example: + 'SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Image File Execution Options\\winword.exe', + default_field: false, + }, + { + name: 'path', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Full path, including hive, key and value', + example: + 'HKLM\\SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Image File Execution\nOptions\\winword.exe\\Debugger', + default_field: false, + }, + { + name: 'value', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the value written.', + example: 'Debugger', + default_field: false, + }, + ], + }, + { + name: 'related', + title: 'Related', + group: 2, + description: + 'This field set is meant to facilitate pivoting around a piece of\ndata.\n\nSome pieces of information can be seen in many places in an ECS event. To facilitate\nsearching for them, store an array of all seen values to their corresponding\nfield in `related.`.\n\nA concrete example is IP addresses, which can be under host, observer, source,\ndestination, client, server, and network.forwarded_ip. If you append all IPs\nto `related.ip`, you can then search for a given IP trivially, no matter where\nit appeared, by querying `related.ip:192.0.2.15`.', + type: 'group', + fields: [ + { + name: 'hash', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + "All the hashes seen on your event. Populating this field, then\nusing it to search for hashes can help in situations where you're unsure what\nthe hash algorithm is (and therefore which key name to search).", + default_field: false, + }, + { + name: 'ip', + level: 'extended', + type: 'ip', + description: 'All of the IPs seen on your event.', + }, + { + name: 'user', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'All the user names seen on your event.', + default_field: false, + }, + ], + }, + { + name: 'rule', + title: 'Rule', + group: 2, + description: + 'Rule fields are used to capture the specifics of any observer or\nagent rules that generate alerts or other notable events.\n\nExamples of data sources that would populate the rule fields include: network\nadmission control platforms, network or host IDS/IPS, network firewalls, web\napplication firewalls, url filters, endpoint detection and response (EDR) systems,\netc.', + type: 'group', + fields: [ + { + name: 'author', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name, organization, or pseudonym of the author or authors who created\nthe rule used to generate this event.', + example: ['Star-Lord'], + default_field: false, + }, + { + name: 'category', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'A categorization value keyword used by the entity using the rule\nfor detection of this event.', + example: 'Attempted Information Leak', + default_field: false, + }, + { + name: 'description', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'The description of the rule generating the event.', + example: 'Block requests to public DNS over HTTPS / TLS protocols', + default_field: false, + }, + { + name: 'id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'A rule ID that is unique within the scope of an agent, observer,\nor other entity using the rule for detection of this event.', + example: 101, + default_field: false, + }, + { + name: 'license', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the license under which the rule used to generate this\nevent is made available.', + example: 'Apache 2.0', + default_field: false, + }, + { + name: 'name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'The name of the rule or signature generating the event.', + example: 'BLOCK_DNS_over_TLS', + default_field: false, + }, + { + name: 'reference', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Reference URL to additional information about the rule used to\ngenerate this event.\n\nThe URL can point to the vendor documentation about the rule. If that is\nnot available, it can also be a link to a more general page describing this\ntype of alert.', + example: 'https://en.wikipedia.org/wiki/DNS_over_TLS', + default_field: false, + }, + { + name: 'ruleset', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the ruleset, policy, group, or parent category in which\nthe rule used to generate this event is a member.', + example: 'Standard_Protocol_Filters', + default_field: false, + }, + { + name: 'uuid', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'A rule ID that is unique within the scope of a set or group of\nagents, observers, or other entities using the rule for detection of this\nevent.', + example: 1100110011, + default_field: false, + }, + { + name: 'version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'The version / revision of the rule being used for analysis.', + example: 1.1, + default_field: false, + }, + ], + }, + { + name: 'server', + title: 'Server', + group: 2, + description: + 'A Server is defined as the responder in a network connection for\nevents regarding sessions, connections, or bidirectional flow records.\n\nFor TCP events, the server is the receiver of the initial SYN packet(s) of the\nTCP connection. For other protocols, the server is generally the responder in\nthe network transaction. Some systems actually use the term "responder" to refer\nthe server in TCP connections. The server fields describe details about the\nsystem acting as the server in the network event. Server fields are usually\npopulated in conjunction with client fields. Server fields are generally not\npopulated for packet-level events.\n\nClient / server representations can add semantic context to an exchange, which\nis helpful to visualize the data in certain situations. If your context falls\nin that category, you should still ensure that source and destination are filled\nappropriately.', + type: 'group', + fields: [ + { + name: 'address', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Some event server addresses are defined ambiguously. The event\nwill sometimes list an IP, a domain or a unix socket. You should always store\nthe raw address in the `.address` field.\n\nThen it should be duplicated to `.ip` or `.domain`, depending on which one\nit is.', + }, + { + name: 'as.number', + level: 'extended', + type: 'long', + description: + 'Unique number allocated to the autonomous system. The autonomous\nsystem number (ASN) uniquely identifies each network on the Internet.', + example: 15169, + }, + { + name: 'as.organization.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Organization name.', + example: 'Google LLC', + }, + { + name: 'bytes', + level: 'core', + type: 'long', + format: 'bytes', + description: 'Bytes sent from the server to the client.', + example: 184, + }, + { + name: 'domain', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Server domain.', + }, + { + name: 'geo.city_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'City name.', + example: 'Montreal', + }, + { + name: 'geo.continent_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the continent.', + example: 'North America', + }, + { + name: 'geo.country_iso_code', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Country ISO code.', + example: 'CA', + }, + { + name: 'geo.country_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Country name.', + example: 'Canada', + }, + { + name: 'geo.location', + level: 'core', + type: 'geo_point', + description: 'Longitude and latitude.', + example: '{ "lon": -73.614830, "lat": 45.505918 }', + }, + { + name: 'geo.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'User-defined description of a location, at the level of granularity\nthey care about.\n\nCould be the name of their data centers, the floor number, if this describes\na local physical entity, city names.\n\nNot typically used in automated geolocation.', + example: 'boston-dc', + }, + { + name: 'geo.region_iso_code', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Region ISO code.', + example: 'CA-QC', + }, + { + name: 'geo.region_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Region name.', + example: 'Quebec', + }, + { + name: 'ip', + level: 'core', + type: 'ip', + description: + 'IP address of the server.\n\nCan be one or multiple IPv4 or IPv6 addresses.', + }, + { + name: 'mac', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'MAC address of the server.', + }, + { + name: 'nat.ip', + level: 'extended', + type: 'ip', + description: + 'Translated ip of destination based NAT sessions (e.g. internet\nto private DMZ)\n\nTypically used with load balancers, firewalls, or routers.', + }, + { + name: 'nat.port', + level: 'extended', + type: 'long', + format: 'string', + description: + 'Translated port of destination based NAT sessions (e.g. internet\nto private DMZ)\n\nTypically used with load balancers, firewalls, or routers.', + }, + { + name: 'packets', + level: 'core', + type: 'long', + description: 'Packets sent from the server to the client.', + example: 12, + }, + { + name: 'port', + level: 'core', + type: 'long', + format: 'string', + description: 'Port of the server.', + }, + { + name: 'registered_domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The highest registered server domain, stripped of the subdomain.\n\nFor example, the registered domain for "foo.google.com" is "google.com".\n\nThis value can be determined precisely with a list like the public suffix\nlist (http://publicsuffix.org). Trying to approximate this by simply taking\nthe last two labels will not work well for TLDs such as "co.uk".', + example: 'google.com', + }, + { + name: 'top_level_domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The effective top level domain (eTLD), also known as the domain\nsuffix, is the last part of the domain name. For example, the top level domain\nfor google.com is "com".\n\nThis value can be determined precisely with a list like the public suffix\nlist (http://publicsuffix.org). Trying to approximate this by simply taking\nthe last label will not work well for effective TLDs such as "co.uk".', + example: 'co.uk', + }, + { + name: 'user.domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the directory the user is a member of.\n\nFor example, an LDAP or Active Directory domain name.', + }, + { + name: 'user.email', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'User email address.', + }, + { + name: 'user.full_name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: "User's full name, if available.", + example: 'Albert Einstein', + }, + { + name: 'user.group.domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the directory the group is a member of.\n\nFor example, an LDAP or Active Directory domain name.', + }, + { + name: 'user.group.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifier for the group on the system/platform.', + }, + { + name: 'user.group.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the group.', + }, + { + name: 'user.hash', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Unique user hash to correlate information for a user in anonymized\nform.\n\nUseful if `user.id` or `user.name` contain confidential information and cannot\nbe used.', + }, + { + name: 'user.id', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifiers of the user.', + }, + { + name: 'user.name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Short name or login of the user.', + example: 'albert', + }, + ], + }, + { + name: 'service', + title: 'Service', + group: 2, + description: + 'The service fields describe the service for or from which the data\nwas collected.\n\nThese fields help you find and correlate logs for a specific service and version.', + type: 'group', + fields: [ + { + name: 'ephemeral_id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Ephemeral identifier of this service (if one exists).\n\nThis id normally changes across restarts, but `service.id` does not.', + example: '8a4f500f', + }, + { + name: 'id', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'Unique identifier of the running service. If the service is comprised\nof many nodes, the `service.id` should be the same for all nodes.\n\nThis id should uniquely identify the service. This makes it possible to correlate\nlogs and metrics for one specific service, no matter which particular node\nemitted the event.\n\nNote that if you need to see the events from one specific host of the service,\nyou should filter on that `host.name` or `host.id` instead.', + example: 'd37e5ebfe0ae6c4972dbe9f0174a1637bb8247f6', + }, + { + name: 'name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the service data is collected from.\n\nThe name of the service is normally user given. This allows for distributed\nservices that run on multiple hosts to correlate the related instances based\non the name.\n\nIn the case of Elasticsearch the `service.name` could contain the cluster\nname. For Beats the `service.name` is by default a copy of the `service.type`\nfield if no name is specified.', + example: 'elasticsearch-metrics', + }, + { + name: 'node.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of a service node.\n\nThis allows for two nodes of the same service running on the same host to\nbe differentiated. Therefore, `service.node.name` should typically be unique\nacross nodes of a given service.\n\nIn the case of Elasticsearch, the `service.node.name` could contain the unique\nnode name within the Elasticsearch cluster. In cases where the service does not\nhave the concept of a node name, the host name or container name can be used\nto distinguish running instances that make up this service. If those do not\nprovide uniqueness (e.g. multiple instances of the service running on the\nsame host) - the node name can be manually set.', + example: 'instance-0000000016', + }, + { + name: 'state', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Current state of the service.', + }, + { + name: 'type', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'The type of the service data is collected from.\n\nThe type can be used to group and correlate logs and metrics from one service\ntype.\n\nExample: If logs or metrics are collected from Elasticsearch, `service.type`\nwould be `elasticsearch`.', + example: 'elasticsearch', + }, + { + name: 'version', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'Version of the service the data was collected from.\n\nThis allows to look at a data set only for a specific version of a service.', + example: '3.2.4', + }, + ], + }, + { + name: 'source', + title: 'Source', + group: 2, + description: + 'Source fields describe details about the source of a packet/event.\n\nSource fields are usually populated in conjunction with destination fields.', + type: 'group', + fields: [ + { + name: 'address', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Some event source addresses are defined ambiguously. The event\nwill sometimes list an IP, a domain or a unix socket. You should always store\nthe raw address in the `.address` field.\n\nThen it should be duplicated to `.ip` or `.domain`, depending on which one\nit is.', + }, + { + name: 'as.number', + level: 'extended', + type: 'long', + description: + 'Unique number allocated to the autonomous system. The autonomous\nsystem number (ASN) uniquely identifies each network on the Internet.', + example: 15169, + }, + { + name: 'as.organization.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Organization name.', + example: 'Google LLC', + }, + { + name: 'bytes', + level: 'core', + type: 'long', + format: 'bytes', + description: 'Bytes sent from the source to the destination.', + example: 184, + }, + { + name: 'domain', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Source domain.', + }, + { + name: 'geo.city_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'City name.', + example: 'Montreal', + }, + { + name: 'geo.continent_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the continent.', + example: 'North America', + }, + { + name: 'geo.country_iso_code', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Country ISO code.', + example: 'CA', + }, + { + name: 'geo.country_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Country name.', + example: 'Canada', + }, + { + name: 'geo.location', + level: 'core', + type: 'geo_point', + description: 'Longitude and latitude.', + example: '{ "lon": -73.614830, "lat": 45.505918 }', + }, + { + name: 'geo.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'User-defined description of a location, at the level of granularity\nthey care about.\n\nCould be the name of their data centers, the floor number, if this describes\na local physical entity, city names.\n\nNot typically used in automated geolocation.', + example: 'boston-dc', + }, + { + name: 'geo.region_iso_code', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Region ISO code.', + example: 'CA-QC', + }, + { + name: 'geo.region_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Region name.', + example: 'Quebec', + }, + { + name: 'ip', + level: 'core', + type: 'ip', + description: + 'IP address of the source.\n\nCan be one or multiple IPv4 or IPv6 addresses.', + }, + { + name: 'mac', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'MAC address of the source.', + }, + { + name: 'nat.ip', + level: 'extended', + type: 'ip', + description: + 'Translated ip of source based NAT sessions (e.g. internal client\nto internet)\n\nTypically connections traversing load balancers, firewalls, or routers.', + }, + { + name: 'nat.port', + level: 'extended', + type: 'long', + format: 'string', + description: + 'Translated port of source based NAT sessions. (e.g. internal client\nto internet)\n\nTypically used with load balancers, firewalls, or routers.', + }, + { + name: 'packets', + level: 'core', + type: 'long', + description: 'Packets sent from the source to the destination.', + example: 12, + }, + { + name: 'port', + level: 'core', + type: 'long', + format: 'string', + description: 'Port of the source.', + }, + { + name: 'registered_domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The highest registered source domain, stripped of the subdomain.\n\nFor example, the registered domain for "foo.google.com" is "google.com".\n\nThis value can be determined precisely with a list like the public suffix\nlist (http://publicsuffix.org). Trying to approximate this by simply taking\nthe last two labels will not work well for TLDs such as "co.uk".', + example: 'google.com', + }, + { + name: 'top_level_domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The effective top level domain (eTLD), also known as the domain\nsuffix, is the last part of the domain name. For example, the top level domain\nfor google.com is "com".\n\nThis value can be determined precisely with a list like the public suffix\nlist (http://publicsuffix.org). Trying to approximate this by simply taking\nthe last label will not work well for effective TLDs such as "co.uk".', + example: 'co.uk', + }, + { + name: 'user.domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the directory the user is a member of.\n\nFor example, an LDAP or Active Directory domain name.', + }, + { + name: 'user.email', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'User email address.', + }, + { + name: 'user.full_name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: "User's full name, if available.", + example: 'Albert Einstein', + }, + { + name: 'user.group.domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the directory the group is a member of.\n\nFor example, an LDAP or Active Directory domain name.', + }, + { + name: 'user.group.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifier for the group on the system/platform.', + }, + { + name: 'user.group.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the group.', + }, + { + name: 'user.hash', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Unique user hash to correlate information for a user in anonymized\nform.\n\nUseful if `user.id` or `user.name` contain confidential information and cannot\nbe used.', + }, + { + name: 'user.id', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifiers of the user.', + }, + { + name: 'user.name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Short name or login of the user.', + example: 'albert', + }, + ], + }, + { + name: 'threat', + title: 'Threat', + group: 2, + description: + 'Fields to classify events and alerts according to a threat taxonomy\nsuch as the Mitre ATT&CK framework.\n\nThese fields are for users to classify alerts from all of their sources (e.g.\nIDS, NGFW, etc.) within a common taxonomy. The threat.tactic.* are meant to\ncapture the high level category of the threat (e.g. "impact"). The threat.technique.*\nfields are meant to capture which kind of approach is used by this detected\nthreat, to accomplish the goal (e.g. "endpoint denial of service").', + type: 'group', + fields: [ + { + name: 'framework', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the threat framework used to further categorize and classify\nthe tactic and technique of the reported threat. Framework classification\ncan be provided by detecting systems, evaluated at ingest time, or retrospectively\ntagged to events.', + example: 'MITRE ATT&CK', + }, + { + name: 'tactic.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The id of tactic used by this threat. You can use the Mitre ATT&CK\nMatrix Tactic categorization, for example. (ex. https://attack.mitre.org/tactics/TA0040/\n)', + example: 'TA0040', + }, + { + name: 'tactic.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the type of tactic used by this threat. You can use the\nMitre ATT&CK Matrix Tactic categorization, for example. (ex. https://attack.mitre.org/tactics/TA0040/\n)', + example: 'impact', + }, + { + name: 'tactic.reference', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The reference url of tactic used by this threat. You can use the\nMitre ATT&CK Matrix Tactic categorization, for example. (ex. https://attack.mitre.org/tactics/TA0040/\n)', + example: 'https://attack.mitre.org/tactics/TA0040/', + }, + { + name: 'technique.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The id of technique used by this tactic. You can use the Mitre\nATT&CK Matrix Tactic categorization, for example. (ex. https://attack.mitre.org/techniques/T1499/\n)', + example: 'T1499', + }, + { + name: 'technique.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: + 'The name of technique used by this tactic. You can use the Mitre\nATT&CK Matrix Tactic categorization, for example. (ex. https://attack.mitre.org/techniques/T1499/\n)', + example: 'endpoint denial of service', + }, + { + name: 'technique.reference', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The reference url of technique used by this tactic. You can use\nthe Mitre ATT&CK Matrix Tactic categorization, for example. (ex. https://attack.mitre.org/techniques/T1499/\n)', + example: 'https://attack.mitre.org/techniques/T1499/', + }, + ], + }, + { + name: 'tls', + title: 'TLS', + group: 2, + description: + 'Fields related to a TLS connection. These fields focus on the TLS\nprotocol itself and intentionally avoids in-depth analysis of the related x.509\ncertificate files.', + type: 'group', + fields: [ + { + name: 'cipher', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'String indicating the cipher used during the current connection.', + example: 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256', + default_field: false, + }, + { + name: 'client.certificate', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'PEM-encoded stand-alone certificate offered by the client. This\nis usually mutually-exclusive of `client.certificate_chain` since this value\nalso exists in that list.', + example: 'MII...', + default_field: false, + }, + { + name: 'client.certificate_chain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Array of PEM-encoded certificates that make up the certificate\nchain offered by the client. This is usually mutually-exclusive of `client.certificate`\nsince that value should be the first certificate in the chain.', + example: ['MII...', 'MII...'], + default_field: false, + }, + { + name: 'client.hash.md5', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Certificate fingerprint using the MD5 digest of DER-encoded version\nof certificate offered by the client. For consistency with other hash values,\nthis value should be formatted as an uppercase hash.', + example: '0F76C7F2C55BFD7D8E8B8F4BFBF0C9EC', + default_field: false, + }, + { + name: 'client.hash.sha1', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Certificate fingerprint using the SHA1 digest of DER-encoded version\nof certificate offered by the client. For consistency with other hash values,\nthis value should be formatted as an uppercase hash.', + example: '9E393D93138888D288266C2D915214D1D1CCEB2A', + default_field: false, + }, + { + name: 'client.hash.sha256', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Certificate fingerprint using the SHA256 digest of DER-encoded\nversion of certificate offered by the client. For consistency with other hash\nvalues, this value should be formatted as an uppercase hash.', + example: '0687F666A054EF17A08E2F2162EAB4CBC0D265E1D7875BE74BF3C712CA92DAF0', + default_field: false, + }, + { + name: 'client.issuer', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Distinguished name of subject of the issuer of the x.509 certificate\npresented by the client.', + example: 'CN=MyDomain Root CA, OU=Infrastructure Team, DC=mydomain, DC=com', + default_field: false, + }, + { + name: 'client.ja3', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'A hash that identifies clients based on how they perform an SSL/TLS\nhandshake.', + example: 'd4e5b18d6b55c71272893221c96ba240', + default_field: false, + }, + { + name: 'client.not_after', + level: 'extended', + type: 'date', + description: + 'Date/Time indicating when client certificate is no longer considered\nvalid.', + example: '2021-01-01T00:00:00.000Z', + default_field: false, + }, + { + name: 'client.not_before', + level: 'extended', + type: 'date', + description: 'Date/Time indicating when client certificate is first considered\nvalid.', + example: '1970-01-01T00:00:00.000Z', + default_field: false, + }, + { + name: 'client.server_name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Also called an SNI, this tells the server which hostname to which\nthe client is attempting to connect. When this value is available, it should\nget copied to `destination.domain`.', + example: 'www.elastic.co', + default_field: false, + }, + { + name: 'client.subject', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Distinguished name of subject of the x.509 certificate presented\nby the client.', + example: 'CN=myclient, OU=Documentation Team, DC=mydomain, DC=com', + default_field: false, + }, + { + name: 'client.supported_ciphers', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Array of ciphers offered by the client during the client hello.', + example: [ + 'TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384', + 'TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384', + '...', + ], + default_field: false, + }, + { + name: 'curve', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'String indicating the curve used for the given cipher, when applicable.', + example: 'secp256r1', + default_field: false, + }, + { + name: 'established', + level: 'extended', + type: 'boolean', + description: + 'Boolean flag indicating if the TLS negotiation was successful and\ntransitioned to an encrypted tunnel.', + default_field: false, + }, + { + name: 'next_protocol', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'String indicating the protocol being tunneled. Per the values in\nthe IANA registry (https://www.iana.org/assignments/tls-extensiontype-values/tls-extensiontype-values.xhtml#alpn-protocol-ids),\nthis string should be lower case.', + example: 'http/1.1', + default_field: false, + }, + { + name: 'resumed', + level: 'extended', + type: 'boolean', + description: + 'Boolean flag indicating if this TLS connection was resumed from\nan existing TLS negotiation.', + default_field: false, + }, + { + name: 'server.certificate', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'PEM-encoded stand-alone certificate offered by the server. This\nis usually mutually-exclusive of `server.certificate_chain` since this value\nalso exists in that list.', + example: 'MII...', + default_field: false, + }, + { + name: 'server.certificate_chain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Array of PEM-encoded certificates that make up the certificate\nchain offered by the server. This is usually mutually-exclusive of `server.certificate`\nsince that value should be the first certificate in the chain.', + example: ['MII...', 'MII...'], + default_field: false, + }, + { + name: 'server.hash.md5', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Certificate fingerprint using the MD5 digest of DER-encoded version\nof certificate offered by the server. For consistency with other hash values,\nthis value should be formatted as an uppercase hash.', + example: '0F76C7F2C55BFD7D8E8B8F4BFBF0C9EC', + default_field: false, + }, + { + name: 'server.hash.sha1', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Certificate fingerprint using the SHA1 digest of DER-encoded version\nof certificate offered by the server. For consistency with other hash values,\nthis value should be formatted as an uppercase hash.', + example: '9E393D93138888D288266C2D915214D1D1CCEB2A', + default_field: false, + }, + { + name: 'server.hash.sha256', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Certificate fingerprint using the SHA256 digest of DER-encoded\nversion of certificate offered by the server. For consistency with other hash\nvalues, this value should be formatted as an uppercase hash.', + example: '0687F666A054EF17A08E2F2162EAB4CBC0D265E1D7875BE74BF3C712CA92DAF0', + default_field: false, + }, + { + name: 'server.issuer', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Subject of the issuer of the x.509 certificate presented by the\nserver.', + example: 'CN=MyDomain Root CA, OU=Infrastructure Team, DC=mydomain, DC=com', + default_field: false, + }, + { + name: 'server.ja3s', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'A hash that identifies servers based on how they perform an SSL/TLS\nhandshake.', + example: '394441ab65754e2207b1e1b457b3641d', + default_field: false, + }, + { + name: 'server.not_after', + level: 'extended', + type: 'date', + description: + 'Timestamp indicating when server certificate is no longer considered\nvalid.', + example: '2021-01-01T00:00:00.000Z', + default_field: false, + }, + { + name: 'server.not_before', + level: 'extended', + type: 'date', + description: 'Timestamp indicating when server certificate is first considered\nvalid.', + example: '1970-01-01T00:00:00.000Z', + default_field: false, + }, + { + name: 'server.subject', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Subject of the x.509 certificate presented by the server.', + example: 'CN=www.mydomain.com, OU=Infrastructure Team, DC=mydomain, DC=com', + default_field: false, + }, + { + name: 'version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Numeric part of the version parsed from the original string.', + example: '1.2', + default_field: false, + }, + { + name: 'version_protocol', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Normalized lowercase protocol name parsed from original string.', + example: 'tls', + default_field: false, + }, + ], + }, + { + name: 'tracing', + title: 'Tracing', + group: 2, + description: + 'Distributed tracing makes it possible to analyze performance throughout\na microservice architecture all in one view. This is accomplished by tracing\nall of the requests - from the initial web request in the front-end service\n- to queries made through multiple back-end services.', + type: 'group', + fields: [ + { + name: 'trace.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Unique identifier of the trace.\n\nA trace groups multiple events like transactions that belong together. For\nexample, a user request handled by multiple inter-connected services.', + example: '4bf92f3577b34da6a3ce929d0e0e4736', + }, + { + name: 'transaction.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Unique identifier of the transaction.\n\nA transaction is the highest level of work measured within a service, such\nas a request to a server.', + example: '00f067aa0ba902b7', + }, + ], + }, + { + name: 'url', + title: 'URL', + group: 2, + description: + 'URL fields provide support for complete or partial URLs, and supports\nthe breaking down into scheme, domain, path, and so on.', + type: 'group', + fields: [ + { + name: 'domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Domain of the url, such as "www.elastic.co".\n\nIn some cases a URL may refer to an IP and/or port directly, without a domain\nname. In this case, the IP address would go to the `domain` field.', + example: 'www.elastic.co', + }, + { + name: 'extension', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The field contains the file extension from the original request\nurl.\n\nThe file extension is only set if it exists, as not every url has a file extension.\n\nThe leading period must not be included. For example, the value must be "png",\nnot ".png".', + example: 'png', + }, + { + name: 'fragment', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Portion of the url after the `#`, such as "top".\n\nThe `#` is not part of the fragment.', + }, + { + name: 'full', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: + 'If full URLs are important to your use case, they should be stored\nin `url.full`, whether this field is reconstructed or present in the event\nsource.', + example: 'https://www.elastic.co:443/search?q=elasticsearch#top', + }, + { + name: 'original', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: + 'Unmodified original url as seen in the event source.\n\nNote that in network monitoring, the observed URL may be a full URL, whereas\nin access logs, the URL is often just represented as a path.\n\nThis field is meant to represent the URL as it was observed, complete or not.', + example: + 'https://www.elastic.co:443/search?q=elasticsearch#top or /search?q=elasticsearch', + }, + { + name: 'password', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Password of the request.', + }, + { + name: 'path', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Path of the request, such as "/search".', + }, + { + name: 'port', + level: 'extended', + type: 'long', + format: 'string', + description: 'Port of the request, such as 443.', + example: 443, + }, + { + name: 'query', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The query field describes the query string of the request, such\nas "q=elasticsearch".\n\nThe `?` is excluded from the query string. If a URL contains no `?`, there\nis no query field. If there is a `?` but no query, the query field exists\nwith an empty string. The `exists` query can be used to differentiate between\nthe two cases.', + }, + { + name: 'registered_domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The highest registered url domain, stripped of the subdomain.\n\nFor example, the registered domain for "foo.google.com" is "google.com".\n\nThis value can be determined precisely with a list like the public suffix\nlist (http://publicsuffix.org). Trying to approximate this by simply taking\nthe last two labels will not work well for TLDs such as "co.uk".', + example: 'google.com', + }, + { + name: 'scheme', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Scheme of the request, such as "https".\n\nNote: The `:` is not part of the scheme.', + example: 'https', + }, + { + name: 'top_level_domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The effective top level domain (eTLD), also known as the domain\nsuffix, is the last part of the domain name. For example, the top level domain\nfor google.com is "com".\n\nThis value can be determined precisely with a list like the public suffix\nlist (http://publicsuffix.org). Trying to approximate this by simply taking\nthe last label will not work well for effective TLDs such as "co.uk".', + example: 'co.uk', + }, + { + name: 'username', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Username of the request.', + }, + ], + }, + { + name: 'user', + title: 'User', + group: 2, + description: + 'The user fields describe information about the user that is relevant\nto the event.\n\nFields can have one entry or multiple entries. If a user has more than one id,\nprovide an array that includes all of them.', + type: 'group', + fields: [ + { + name: 'domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the directory the user is a member of.\n\nFor example, an LDAP or Active Directory domain name.', + }, + { + name: 'email', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'User email address.', + }, + { + name: 'full_name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: "User's full name, if available.", + example: 'Albert Einstein', + }, + { + name: 'group.domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the directory the group is a member of.\n\nFor example, an LDAP or Active Directory domain name.', + }, + { + name: 'group.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifier for the group on the system/platform.', + }, + { + name: 'group.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the group.', + }, + { + name: 'hash', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Unique user hash to correlate information for a user in anonymized\nform.\n\nUseful if `user.id` or `user.name` contain confidential information and cannot\nbe used.', + }, + { + name: 'id', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifiers of the user.', + }, + { + name: 'name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Short name or login of the user.', + example: 'albert', + }, + ], + }, + { + name: 'user_agent', + title: 'User agent', + group: 2, + description: + 'The user_agent fields normally come from a browser request.\n\nThey often show up in web service logs coming from the parsed user agent string.', + type: 'group', + fields: [ + { + name: 'device.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the device.', + example: 'iPhone', + }, + { + name: 'name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the user agent.', + example: 'Safari', + }, + { + name: 'original', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + }, + ], + description: 'Unparsed user_agent string.', + example: + 'Mozilla/5.0 (iPhone; CPU iPhone OS 12_1 like Mac OS X) AppleWebKit/605.1.15\n(KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1', + }, + { + name: 'os.family', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'OS family (such as redhat, debian, freebsd, windows).', + example: 'debian', + }, + { + name: 'os.full', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Operating system name, including the version or code name.', + example: 'Mac OS Mojave', + }, + { + name: 'os.kernel', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Operating system kernel version as a raw string.', + example: '4.4.0-112-generic', + }, + { + name: 'os.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Operating system name, without the version.', + example: 'Mac OS X', + }, + { + name: 'os.platform', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Operating system platform (such centos, ubuntu, windows).', + example: 'darwin', + }, + { + name: 'os.version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Operating system version as a raw string.', + example: '10.14.1', + }, + { + name: 'version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Version of the user agent.', + example: 12, + }, + ], + }, + { + name: 'vlan', + title: 'VLAN', + group: 2, + description: + 'The VLAN fields are used to identify 802.1q tag(s) of a packet,\nas well as ingress and egress VLAN associations of an observer in relation to\na specific packet or connection.\n\nNetwork.vlan fields are used to record a single VLAN tag, or the outer tag in\nthe case of q-in-q encapsulations, for a packet or connection as observed, typically\nprovided by a network sensor (e.g. Zeek, Wireshark) passively reporting on traffic.\n\nNetwork.inner VLAN fields are used to report inner q-in-q 802.1q tags (multiple\n802.1q encapsulations) as observed, typically provided by a network sensor (e.g.\nZeek, Wireshark) passively reporting on traffic. Network.inner VLAN fields should\nonly be used in addition to network.vlan fields to indicate q-in-q tagging.\n\nObserver.ingress and observer.egress VLAN values are used to record observer\nspecific information when observer events contain discrete ingress and egress\nVLAN information, typically provided by firewalls, routers, or load balancers.', + type: 'group', + fields: [ + { + name: 'id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'VLAN ID as reported by the observer.', + example: 10, + default_field: false, + }, + { + name: 'name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Optional VLAN name as reported by the observer.', + example: 'outside', + default_field: false, + }, + ], + }, + { + name: 'vulnerability', + title: 'Vulnerability', + group: 2, + description: + 'The vulnerability fields describe information about a vulnerability\nthat is relevant to an event.', + type: 'group', + fields: [ + { + name: 'category', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The type of system or architecture that the vulnerability affects.\nThese may be platform-specific (for example, Debian or SUSE) or general (for\nexample, Database or Firewall). For example (https://qualysguard.qualys.com/qwebhelp/fo_portal/knowledgebase/vulnerability_categories.htm[Qualys\nvulnerability categories])\n\nThis field must be an array.', + example: '["Firewall"]', + default_field: false, + }, + { + name: 'classification', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The classification of the vulnerability scoring system. For example\n(https://www.first.org/cvss/)', + example: 'CVSS', + default_field: false, + }, + { + name: 'description', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + }, + ], + description: + 'The description of the vulnerability that provides additional context\nof the vulnerability. For example (https://cve.mitre.org/about/faqs.html#cve_entry_descriptions_created[Common\nVulnerabilities and Exposure CVE description])', + example: 'In macOS before 2.12.6, there is a vulnerability in the RPC...', + default_field: false, + }, + { + name: 'enumeration', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The type of identifier used for this vulnerability. For example\n(https://cve.mitre.org/about/)', + example: 'CVE', + default_field: false, + }, + { + name: 'id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The identification (ID) is the number portion of a vulnerability\nentry. It includes a unique identification number for the vulnerability. For\nexample (https://cve.mitre.org/about/faqs.html#what_is_cve_id)[Common Vulnerabilities\nand Exposure CVE ID]', + example: 'CVE-2019-00001', + default_field: false, + }, + { + name: 'reference', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'A resource that provides additional information, context, and mitigations\nfor the identified vulnerability.', + example: 'https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-6111', + default_field: false, + }, + { + name: 'report_id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'The report or scan identification number.', + example: 20191018.0001, + default_field: false, + }, + { + name: 'scanner.vendor', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'The name of the vulnerability scanner vendor.', + example: 'Tenable', + default_field: false, + }, + { + name: 'score.base', + level: 'extended', + type: 'float', + description: + 'Scores can range from 0.0 to 10.0, with 10.0 being the most severe.\n\nBase scores cover an assessment for exploitability metrics (attack vector,\ncomplexity, privileges, and user interaction), impact metrics (confidentiality,\nintegrity, and availability), and scope. For example (https://www.first.org/cvss/specification-document)', + example: 5.5, + default_field: false, + }, + { + name: 'score.environmental', + level: 'extended', + type: 'float', + description: + 'Scores can range from 0.0 to 10.0, with 10.0 being the most severe.\n\nEnvironmental scores cover an assessment for any modified Base metrics, confidentiality,\nintegrity, and availability requirements. For example (https://www.first.org/cvss/specification-document)', + example: 5.5, + default_field: false, + }, + { + name: 'score.temporal', + level: 'extended', + type: 'float', + description: + 'Scores can range from 0.0 to 10.0, with 10.0 being the most severe.\n\nTemporal scores cover an assessment for code maturity, remediation level,\nand confidence. For example (https://www.first.org/cvss/specification-document)', + default_field: false, + }, + { + name: 'score.version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The National Vulnerability Database (NVD) provides qualitative\nseverity rankings of "Low", "Medium", and "High" for CVSS v2.0 base score\nranges in addition to the severity ratings for CVSS v3.0 as they are defined\nin the CVSS v3.0 specification.\n\nCVSS is owned and managed by FIRST.Org, Inc. (FIRST), a US-based non-profit\norganization, whose mission is to help computer security incident response\nteams across the world. For example (https://nvd.nist.gov/vuln-metrics/cvss)', + example: 2, + default_field: false, + }, + { + name: 'severity', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The severity of the vulnerability can help with metrics and internal\nprioritization regarding remediation. For example (https://nvd.nist.gov/vuln-metrics/cvss)', + example: 'Critical', + default_field: false, + }, + ], + }, + ], + }, + { + key: 'beat', + anchor: 'beat-common', + title: 'Beat', + description: 'Contains common beat fields available in all event types.\n', + fields: [ + { + name: 'agent.hostname', + type: 'keyword', + description: 'Hostname of the agent.', + }, + { + name: 'beat.timezone', + type: 'alias', + path: 'event.timezone', + migration: true, + }, + { + name: 'fields', + type: 'object', + object_type: 'keyword', + description: 'Contains user configurable fields.\n', + }, + { + name: 'beat.name', + type: 'alias', + path: 'host.name', + migration: true, + }, + { + name: 'beat.hostname', + type: 'alias', + path: 'agent.hostname', + migration: true, + }, + { + name: 'timeseries.instance', + type: 'keyword', + description: 'Time series instance id', + }, + ], + }, + { + key: 'cloud', + title: 'Cloud provider metadata', + description: 'Metadata from cloud providers added by the add_cloud_metadata processor.\n', + fields: [ + { + name: 'cloud.project.id', + example: 'project-x', + description: 'Name of the project in Google Cloud.\n', + }, + { + name: 'cloud.image.id', + example: 'ami-abcd1234', + description: 'Image ID for the cloud instance.\n', + }, + { + name: 'meta.cloud.provider', + type: 'alias', + path: 'cloud.provider', + migration: true, + }, + { + name: 'meta.cloud.instance_id', + type: 'alias', + path: 'cloud.instance.id', + migration: true, + }, + { + name: 'meta.cloud.instance_name', + type: 'alias', + path: 'cloud.instance.name', + migration: true, + }, + { + name: 'meta.cloud.machine_type', + type: 'alias', + path: 'cloud.machine.type', + migration: true, + }, + { + name: 'meta.cloud.availability_zone', + type: 'alias', + path: 'cloud.availability_zone', + migration: true, + }, + { + name: 'meta.cloud.project_id', + type: 'alias', + path: 'cloud.project.id', + migration: true, + }, + { + name: 'meta.cloud.region', + type: 'alias', + path: 'cloud.region', + migration: true, + }, + ], + }, + { + key: 'docker', + title: 'Docker', + description: 'Docker stats collected from Docker.\n', + short_config: false, + anchor: 'docker-processor', + fields: [ + { + name: 'docker', + type: 'group', + fields: [ + { + name: 'container.id', + type: 'alias', + path: 'container.id', + migration: true, + }, + { + name: 'container.image', + type: 'alias', + path: 'container.image.name', + migration: true, + }, + { + name: 'container.name', + type: 'alias', + path: 'container.name', + migration: true, + }, + { + name: 'container.labels', + type: 'object', + object_type: 'keyword', + description: 'Image labels.\n', + }, + ], + }, + ], + }, + { + key: 'host', + title: 'Host', + description: 'Info collected for the host machine.\n', + anchor: 'host-processor', + fields: [ + { + name: 'host', + type: 'group', + fields: [ + { + name: 'containerized', + type: 'boolean', + description: 'If the host is a container.\n', + }, + { + name: 'os.build', + type: 'keyword', + example: '18D109', + description: 'OS build information.\n', + }, + { + name: 'os.codename', + type: 'keyword', + example: 'stretch', + description: 'OS codename, if any.\n', + }, + ], + }, + ], + }, + { + key: 'kubernetes', + title: 'Kubernetes', + description: 'Kubernetes metadata added by the kubernetes processor\n', + short_config: false, + anchor: 'kubernetes-processor', + fields: [ + { + name: 'kubernetes', + type: 'group', + fields: [ + { + name: 'pod.name', + type: 'keyword', + description: 'Kubernetes pod name\n', + }, + { + name: 'pod.uid', + type: 'keyword', + description: 'Kubernetes Pod UID\n', + }, + { + name: 'namespace', + type: 'keyword', + description: 'Kubernetes namespace\n', + }, + { + name: 'node.name', + type: 'keyword', + description: 'Kubernetes node name\n', + }, + { + name: 'labels.*', + type: 'object', + object_type: 'keyword', + object_type_mapping_type: '*', + description: 'Kubernetes labels map\n', + }, + { + name: 'annotations.*', + type: 'object', + object_type: 'keyword', + object_type_mapping_type: '*', + description: 'Kubernetes annotations map\n', + }, + { + name: 'replicaset.name', + type: 'keyword', + description: 'Kubernetes replicaset name\n', + }, + { + name: 'deployment.name', + type: 'keyword', + description: 'Kubernetes deployment name\n', + }, + { + name: 'statefulset.name', + type: 'keyword', + description: 'Kubernetes statefulset name\n', + }, + { + name: 'container.name', + type: 'keyword', + description: 'Kubernetes container name\n', + }, + { + name: 'container.image', + type: 'keyword', + description: 'Kubernetes container image\n', + }, + ], + }, + ], + }, + { + key: 'process', + title: 'Process', + description: 'Process metadata fields\n', + fields: [ + { + name: 'process', + type: 'group', + fields: [ + { + name: 'exe', + type: 'alias', + path: 'process.executable', + migration: true, + }, + ], + }, + ], + }, + { + key: 'jolokia-autodiscover', + title: 'Jolokia Discovery autodiscover provider', + description: 'Metadata from Jolokia Discovery added by the jolokia provider.\n', + fields: [ + { + name: 'jolokia.agent.version', + type: 'keyword', + description: 'Version number of jolokia agent.\n', + }, + { + name: 'jolokia.agent.id', + type: 'keyword', + description: + 'Each agent has a unique id which can be either provided during startup of the agent in form of a configuration parameter or being autodetected. If autodected, the id has several parts: The IP, the process id, hashcode of the agent and its type.\n', + }, + { + name: 'jolokia.server.product', + type: 'keyword', + description: 'The container product if detected.\n', + }, + { + name: 'jolokia.server.version', + type: 'keyword', + description: "The container's version (if detected).\n", + }, + { + name: 'jolokia.server.vendor', + type: 'keyword', + description: 'The vendor of the container the agent is running in.\n', + }, + { + name: 'jolokia.url', + type: 'keyword', + description: 'The URL how this agent can be contacted.\n', + }, + { + name: 'jolokia.secured', + type: 'boolean', + description: 'Whether the agent was configured for authentication or not.\n', + }, + ], + }, + { + key: 'common', + title: 'Common', + description: 'Contains common fields available in all event types.\n', + fields: [ + { + name: 'file', + type: 'group', + description: 'File attributes.', + fields: [ + { + name: 'setuid', + type: 'boolean', + example: true, + description: 'Set if the file has the `setuid` bit set. Omitted otherwise.', + }, + { + name: 'setgid', + type: 'boolean', + example: true, + description: 'Set if the file has the `setgid` bit set. Omitted otherwise.', + }, + { + name: 'origin', + type: 'keyword', + description: + 'An array of strings describing a possible external origin for this file. For example, the URL it was downloaded from. Only supported in macOS, via the kMDItemWhereFroms attribute. Omitted if origin information is not available.\n', + multi_fields: [ + { + name: 'raw', + type: 'keyword', + description: + 'This is a non-analyzed field that is useful for aggregations on the origin data.\n', + }, + ], + }, + { + name: 'selinux', + type: 'group', + description: 'The SELinux identity of the file.', + fields: [ + { + name: 'user', + type: 'keyword', + description: 'The owner of the object.', + }, + { + name: 'role', + type: 'keyword', + description: "The object's SELinux role.", + }, + { + name: 'domain', + type: 'keyword', + description: "The object's SELinux domain or type.", + }, + { + name: 'level', + type: 'keyword', + example: 's0', + description: "The object's SELinux level.", + }, + ], + }, + ], + }, + { + name: 'user', + type: 'group', + description: 'User information.', + fields: [ + { + name: 'audit', + type: 'group', + description: 'Audit user information.', + fields: [ + { + name: 'id', + type: 'keyword', + description: 'Audit user ID.', + }, + { + name: 'name', + type: 'keyword', + description: 'Audit user name.', + }, + ], + }, + { + name: 'effective', + type: 'group', + description: 'Effective user information.', + fields: [ + { + name: 'id', + type: 'keyword', + description: 'Effective user ID.', + }, + { + name: 'name', + type: 'keyword', + description: 'Effective user name.', + }, + { + name: 'group', + type: 'group', + description: 'Effective group information.', + fields: [ + { + name: 'id', + type: 'keyword', + description: 'Effective group ID.', + }, + { + name: 'name', + type: 'keyword', + description: 'Effective group name.', + }, + ], + }, + ], + }, + { + name: 'filesystem', + type: 'group', + description: 'Filesystem user information.', + fields: [ + { + name: 'id', + type: 'keyword', + description: 'Filesystem user ID.', + }, + { + name: 'name', + type: 'keyword', + description: 'Filesystem user name.', + }, + { + name: 'group', + type: 'group', + description: 'Filesystem group information.', + fields: [ + { + name: 'id', + type: 'keyword', + description: 'Filesystem group ID.', + }, + { + name: 'name', + type: 'keyword', + description: 'Filesystem group name.', + }, + ], + }, + ], + }, + { + name: 'saved', + type: 'group', + description: 'Saved user information.', + fields: [ + { + name: 'id', + type: 'keyword', + description: 'Saved user ID.', + }, + { + name: 'name', + type: 'keyword', + description: 'Saved user name.', + }, + { + name: 'group', + type: 'group', + description: 'Saved group information.', + fields: [ + { + name: 'id', + type: 'keyword', + description: 'Saved group ID.', + }, + { + name: 'name', + type: 'keyword', + description: 'Saved group name.', + }, + ], + }, + ], + }, + ], + }, + ], + }, + { + key: 'auditd', + title: 'Auditd', + description: 'These are the fields generated by the auditd module.', + fields: [ + { + name: 'user', + type: 'group', + fields: [ + { + name: 'auid', + type: 'alias', + path: 'user.audit.id', + migration: true, + }, + { + name: 'uid', + type: 'alias', + path: 'user.id', + migration: true, + }, + { + name: 'euid', + type: 'alias', + path: 'user.effective.id', + migration: true, + }, + { + name: 'fsuid', + type: 'alias', + path: 'user.filesystem.id', + migration: true, + }, + { + name: 'suid', + type: 'alias', + path: 'user.saved.id', + migration: true, + }, + { + name: 'gid', + type: 'alias', + path: 'user.group.id', + migration: true, + }, + { + name: 'egid', + type: 'alias', + path: 'user.effective.group.id', + migration: true, + }, + { + name: 'sgid', + type: 'alias', + path: 'user.saved.group.id', + migration: true, + }, + { + name: 'fsgid', + type: 'alias', + path: 'user.filesystem.group.id', + migration: true, + }, + { + name: 'name_map', + type: 'group', + description: + 'If `resolve_ids` is set to true in the configuration then `name_map` will contain a mapping of uid field names to the resolved name (e.g. auid -> root).\n', + fields: [ + { + name: 'auid', + type: 'alias', + path: 'user.audit.name', + migration: true, + }, + { + name: 'uid', + type: 'alias', + path: 'user.name', + migration: true, + }, + { + name: 'euid', + type: 'alias', + path: 'user.effective.name', + migration: true, + }, + { + name: 'fsuid', + type: 'alias', + path: 'user.filesystem.name', + migration: true, + }, + { + name: 'suid', + type: 'alias', + path: 'user.saved.name', + migration: true, + }, + { + name: 'gid', + type: 'alias', + path: 'user.group.name', + migration: true, + }, + { + name: 'egid', + type: 'alias', + path: 'user.effective.group.name', + migration: true, + }, + { + name: 'sgid', + type: 'alias', + path: 'user.saved.group.name', + migration: true, + }, + { + name: 'fsgid', + type: 'alias', + path: 'user.filesystem.group.name', + migration: true, + }, + ], + }, + { + name: 'selinux', + type: 'group', + description: 'The SELinux identity of the actor.', + fields: [ + { + name: 'user', + type: 'keyword', + description: 'account submitted for authentication', + }, + { + name: 'role', + type: 'keyword', + description: "user's SELinux role", + }, + { + name: 'domain', + type: 'keyword', + description: "The actor's SELinux domain or type.", + }, + { + name: 'level', + type: 'keyword', + example: 's0', + description: "The actor's SELinux level.", + }, + { + name: 'category', + type: 'keyword', + description: "The actor's SELinux category or compartments.", + }, + ], + }, + ], + }, + { + name: 'process', + type: 'group', + description: 'Process attributes.', + fields: [ + { + name: 'cwd', + type: 'alias', + path: 'process.working_directory', + migration: true, + description: 'The current working directory.', + }, + ], + }, + { + name: 'source', + type: 'group', + description: 'Source that triggered the event.', + fields: [ + { + name: 'path', + type: 'keyword', + description: 'This is the path associated with a unix socket.', + }, + ], + }, + { + name: 'destination', + type: 'group', + description: 'Destination address that triggered the event.', + fields: [ + { + name: 'path', + type: 'keyword', + description: 'This is the path associated with a unix socket.', + }, + ], + }, + { + name: 'auditd', + type: 'group', + fields: [ + { + name: 'message_type', + type: 'keyword', + example: 'syscall', + description: 'The audit message type (e.g. syscall or apparmor_denied).\n', + }, + { + name: 'sequence', + type: 'long', + description: + 'The sequence number of the event as assigned by the kernel. Sequence numbers are stored as a uint32 in the kernel and can rollover.\n', + }, + { + name: 'session', + type: 'keyword', + description: + 'The session ID assigned to a login. All events related to a login session will have the same value.\n', + }, + { + name: 'result', + type: 'keyword', + example: 'success or fail', + description: 'The result of the audited operation (success/fail).', + }, + { + name: 'summary', + type: 'group', + fields: [ + { + name: 'actor', + type: 'group', + description: 'The actor is the user that triggered the audit event.', + fields: [ + { + name: 'primary', + type: 'keyword', + description: + "The primary identity of the actor. This is the actor's original login ID. It will not change even if the user changes to another account.\n", + }, + { + name: 'secondary', + type: 'keyword', + description: + 'The secondary identity of the actor. This is typically\nthe same as the primary, except for when the user has used `su`.', + }, + ], + }, + { + name: 'object', + type: 'group', + description: 'This is the thing or object being acted upon in the event.\n', + fields: [ + { + name: 'type', + type: 'keyword', + description: + 'A description of the what the "thing" is (e.g. file, socket, user-session).\n', + }, + { + name: 'primary', + type: 'keyword', + description: '', + }, + { + name: 'secondary', + type: 'keyword', + description: '', + }, + ], + }, + { + name: 'how', + type: 'keyword', + description: + 'This describes how the action was performed. Usually this is the exe or command that was being executed that triggered the event.\n', + }, + ], + }, + { + name: 'paths', + type: 'group', + description: 'List of paths associated with the event.', + fields: [ + { + name: 'inode', + type: 'keyword', + description: 'inode number', + }, + { + name: 'dev', + type: 'keyword', + description: 'device name as found in /dev', + }, + { + name: 'obj_user', + type: 'keyword', + description: '', + }, + { + name: 'obj_role', + type: 'keyword', + description: '', + }, + { + name: 'obj_domain', + type: 'keyword', + description: '', + }, + { + name: 'obj_level', + type: 'keyword', + description: '', + }, + { + name: 'objtype', + type: 'keyword', + description: '', + }, + { + name: 'ouid', + type: 'keyword', + description: 'file owner user ID', + }, + { + name: 'rdev', + type: 'keyword', + description: 'the device identifier (special files only)', + }, + { + name: 'nametype', + type: 'keyword', + description: 'kind of file operation being referenced', + }, + { + name: 'ogid', + type: 'keyword', + description: 'file owner group ID', + }, + { + name: 'item', + type: 'keyword', + description: 'which item is being recorded', + }, + { + name: 'mode', + type: 'keyword', + description: 'mode flags on a file', + }, + { + name: 'name', + type: 'keyword', + description: 'file name in avcs', + }, + ], + }, + { + name: 'data', + type: 'group', + description: 'The data from the audit messages.', + fields: [ + { + name: 'action', + type: 'keyword', + description: 'netfilter packet disposition', + }, + { + name: 'minor', + type: 'keyword', + description: 'device minor number', + }, + { + name: 'acct', + type: 'keyword', + description: "a user's account name", + }, + { + name: 'addr', + type: 'keyword', + description: 'the remote address that the user is connecting from', + }, + { + name: 'cipher', + type: 'keyword', + description: 'name of crypto cipher selected', + }, + { + name: 'id', + type: 'keyword', + description: 'during account changes', + }, + { + name: 'entries', + type: 'keyword', + description: 'number of entries in the netfilter table', + }, + { + name: 'kind', + type: 'keyword', + description: 'server or client in crypto operation', + }, + { + name: 'ksize', + type: 'keyword', + description: 'key size for crypto operation', + }, + { + name: 'spid', + type: 'keyword', + description: 'sent process ID', + }, + { + name: 'arch', + type: 'keyword', + description: 'the elf architecture flags', + }, + { + name: 'argc', + type: 'keyword', + description: 'the number of arguments to an execve syscall', + }, + { + name: 'major', + type: 'keyword', + description: 'device major number', + }, + { + name: 'unit', + type: 'keyword', + description: 'systemd unit', + }, + { + name: 'table', + type: 'keyword', + description: 'netfilter table name', + }, + { + name: 'terminal', + type: 'keyword', + description: 'terminal name the user is running programs on', + }, + { + name: 'grantors', + type: 'keyword', + description: 'pam modules approving the action', + }, + { + name: 'direction', + type: 'keyword', + description: 'direction of crypto operation', + }, + { + name: 'op', + type: 'keyword', + description: 'the operation being performed that is audited', + }, + { + name: 'tty', + type: 'keyword', + description: 'tty udevice the user is running programs on', + }, + { + name: 'syscall', + type: 'keyword', + description: 'syscall number in effect when the event occurred', + }, + { + name: 'data', + type: 'keyword', + description: 'TTY text', + }, + { + name: 'family', + type: 'keyword', + description: 'netfilter protocol', + }, + { + name: 'mac', + type: 'keyword', + description: 'crypto MAC algorithm selected', + }, + { + name: 'pfs', + type: 'keyword', + description: 'perfect forward secrecy method', + }, + { + name: 'items', + type: 'keyword', + description: 'the number of path records in the event', + }, + { + name: 'a0', + type: 'keyword', + description: '', + }, + { + name: 'a1', + type: 'keyword', + description: '', + }, + { + name: 'a2', + type: 'keyword', + description: '', + }, + { + name: 'a3', + type: 'keyword', + description: '', + }, + { + name: 'hostname', + type: 'keyword', + description: 'the hostname that the user is connecting from', + }, + { + name: 'lport', + type: 'keyword', + description: 'local network port', + }, + { + name: 'rport', + type: 'keyword', + description: 'remote port number', + }, + { + name: 'exit', + type: 'keyword', + description: 'syscall exit code', + }, + { + name: 'fp', + type: 'keyword', + description: 'crypto key finger print', + }, + { + name: 'laddr', + type: 'keyword', + description: 'local network address', + }, + { + name: 'sport', + type: 'keyword', + description: 'local port number', + }, + { + name: 'capability', + type: 'keyword', + description: 'posix capabilities', + }, + { + name: 'nargs', + type: 'keyword', + description: 'the number of arguments to a socket call', + }, + { + name: 'new-enabled', + type: 'keyword', + description: 'new TTY audit enabled setting', + }, + { + name: 'audit_backlog_limit', + type: 'keyword', + description: "audit system's backlog queue size", + }, + { + name: 'dir', + type: 'keyword', + description: 'directory name', + }, + { + name: 'cap_pe', + type: 'keyword', + description: 'process effective capability map', + }, + { + name: 'model', + type: 'keyword', + description: 'security model being used for virt', + }, + { + name: 'new_pp', + type: 'keyword', + description: 'new process permitted capability map', + }, + { + name: 'old-enabled', + type: 'keyword', + description: 'present TTY audit enabled setting', + }, + { + name: 'oauid', + type: 'keyword', + description: "object's login user ID", + }, + { + name: 'old', + type: 'keyword', + description: 'old value', + }, + { + name: 'banners', + type: 'keyword', + description: 'banners used on printed page', + }, + { + name: 'feature', + type: 'keyword', + description: 'kernel feature being changed', + }, + { + name: 'vm-ctx', + type: 'keyword', + description: "the vm's context string", + }, + { + name: 'opid', + type: 'keyword', + description: "object's process ID", + }, + { + name: 'seperms', + type: 'keyword', + description: 'SELinux permissions being used', + }, + { + name: 'seresult', + type: 'keyword', + description: 'SELinux AVC decision granted/denied', + }, + { + name: 'new-rng', + type: 'keyword', + description: 'device name of rng being added from a vm', + }, + { + name: 'old-net', + type: 'keyword', + description: 'present MAC address assigned to vm', + }, + { + name: 'sigev_signo', + type: 'keyword', + description: 'signal number', + }, + { + name: 'ino', + type: 'keyword', + description: 'inode number', + }, + { + name: 'old_enforcing', + type: 'keyword', + description: 'old MAC enforcement status', + }, + { + name: 'old-vcpu', + type: 'keyword', + description: 'present number of CPU cores', + }, + { + name: 'range', + type: 'keyword', + description: "user's SE Linux range", + }, + { + name: 'res', + type: 'keyword', + description: 'result of the audited operation(success/fail)', + }, + { + name: 'added', + type: 'keyword', + description: 'number of new files detected', + }, + { + name: 'fam', + type: 'keyword', + description: 'socket address family', + }, + { + name: 'nlnk-pid', + type: 'keyword', + description: 'pid of netlink packet sender', + }, + { + name: 'subj', + type: 'keyword', + description: "lspp subject's context string", + }, + { + name: 'a[0-3]', + type: 'keyword', + description: 'the arguments to a syscall', + }, + { + name: 'cgroup', + type: 'keyword', + description: 'path to cgroup in sysfs', + }, + { + name: 'kernel', + type: 'keyword', + description: "kernel's version number", + }, + { + name: 'ocomm', + type: 'keyword', + description: "object's command line name", + }, + { + name: 'new-net', + type: 'keyword', + description: 'MAC address being assigned to vm', + }, + { + name: 'permissive', + type: 'keyword', + description: 'SELinux is in permissive mode', + }, + { + name: 'class', + type: 'keyword', + description: 'resource class assigned to vm', + }, + { + name: 'compat', + type: 'keyword', + description: 'is_compat_task result', + }, + { + name: 'fi', + type: 'keyword', + description: 'file assigned inherited capability map', + }, + { + name: 'changed', + type: 'keyword', + description: 'number of changed files', + }, + { + name: 'msg', + type: 'keyword', + description: 'the payload of the audit record', + }, + { + name: 'dport', + type: 'keyword', + description: 'remote port number', + }, + { + name: 'new-seuser', + type: 'keyword', + description: 'new SELinux user', + }, + { + name: 'invalid_context', + type: 'keyword', + description: 'SELinux context', + }, + { + name: 'dmac', + type: 'keyword', + description: 'remote MAC address', + }, + { + name: 'ipx-net', + type: 'keyword', + description: 'IPX network number', + }, + { + name: 'iuid', + type: 'keyword', + description: "ipc object's user ID", + }, + { + name: 'macproto', + type: 'keyword', + description: 'ethernet packet type ID field', + }, + { + name: 'obj', + type: 'keyword', + description: 'lspp object context string', + }, + { + name: 'ipid', + type: 'keyword', + description: 'IP datagram fragment identifier', + }, + { + name: 'new-fs', + type: 'keyword', + description: 'file system being added to vm', + }, + { + name: 'vm-pid', + type: 'keyword', + description: "vm's process ID", + }, + { + name: 'cap_pi', + type: 'keyword', + description: 'process inherited capability map', + }, + { + name: 'old-auid', + type: 'keyword', + description: 'previous auid value', + }, + { + name: 'oses', + type: 'keyword', + description: "object's session ID", + }, + { + name: 'fd', + type: 'keyword', + description: 'file descriptor number', + }, + { + name: 'igid', + type: 'keyword', + description: "ipc object's group ID", + }, + { + name: 'new-disk', + type: 'keyword', + description: 'disk being added to vm', + }, + { + name: 'parent', + type: 'keyword', + description: 'the inode number of the parent file', + }, + { + name: 'len', + type: 'keyword', + description: 'length', + }, + { + name: 'oflag', + type: 'keyword', + description: 'open syscall flags', + }, + { + name: 'uuid', + type: 'keyword', + description: 'a UUID', + }, + { + name: 'code', + type: 'keyword', + description: 'seccomp action code', + }, + { + name: 'nlnk-grp', + type: 'keyword', + description: 'netlink group number', + }, + { + name: 'cap_fp', + type: 'keyword', + description: 'file permitted capability map', + }, + { + name: 'new-mem', + type: 'keyword', + description: 'new amount of memory in KB', + }, + { + name: 'seperm', + type: 'keyword', + description: 'SELinux permission being decided on', + }, + { + name: 'enforcing', + type: 'keyword', + description: 'new MAC enforcement status', + }, + { + name: 'new-chardev', + type: 'keyword', + description: 'new character device being assigned to vm', + }, + { + name: 'old-rng', + type: 'keyword', + description: 'device name of rng being removed from a vm', + }, + { + name: 'outif', + type: 'keyword', + description: 'out interface number', + }, + { + name: 'cmd', + type: 'keyword', + description: 'command being executed', + }, + { + name: 'hook', + type: 'keyword', + description: 'netfilter hook that packet came from', + }, + { + name: 'new-level', + type: 'keyword', + description: 'new run level', + }, + { + name: 'sauid', + type: 'keyword', + description: 'sent login user ID', + }, + { + name: 'sig', + type: 'keyword', + description: 'signal number', + }, + { + name: 'audit_backlog_wait_time', + type: 'keyword', + description: "audit system's backlog wait time", + }, + { + name: 'printer', + type: 'keyword', + description: 'printer name', + }, + { + name: 'old-mem', + type: 'keyword', + description: 'present amount of memory in KB', + }, + { + name: 'perm', + type: 'keyword', + description: 'the file permission being used', + }, + { + name: 'old_pi', + type: 'keyword', + description: 'old process inherited capability map', + }, + { + name: 'state', + type: 'keyword', + description: 'audit daemon configuration resulting state', + }, + { + name: 'format', + type: 'keyword', + description: "audit log's format", + }, + { + name: 'new_gid', + type: 'keyword', + description: 'new group ID being assigned', + }, + { + name: 'tcontext', + type: 'keyword', + description: "the target's or object's context string", + }, + { + name: 'maj', + type: 'keyword', + description: 'device major number', + }, + { + name: 'watch', + type: 'keyword', + description: 'file name in a watch record', + }, + { + name: 'device', + type: 'keyword', + description: 'device name', + }, + { + name: 'grp', + type: 'keyword', + description: 'group name', + }, + { + name: 'bool', + type: 'keyword', + description: 'name of SELinux boolean', + }, + { + name: 'icmp_type', + type: 'keyword', + description: 'type of icmp message', + }, + { + name: 'new_lock', + type: 'keyword', + description: 'new value of feature lock', + }, + { + name: 'old_prom', + type: 'keyword', + description: 'network promiscuity flag', + }, + { + name: 'acl', + type: 'keyword', + description: 'access mode of resource assigned to vm', + }, + { + name: 'ip', + type: 'keyword', + description: 'network address of a printer', + }, + { + name: 'new_pi', + type: 'keyword', + description: 'new process inherited capability map', + }, + { + name: 'default-context', + type: 'keyword', + description: 'default MAC context', + }, + { + name: 'inode_gid', + type: 'keyword', + description: "group ID of the inode's owner", + }, + { + name: 'new-log_passwd', + type: 'keyword', + description: 'new value for TTY password logging', + }, + { + name: 'new_pe', + type: 'keyword', + description: 'new process effective capability map', + }, + { + name: 'selected-context', + type: 'keyword', + description: 'new MAC context assigned to session', + }, + { + name: 'cap_fver', + type: 'keyword', + description: 'file system capabilities version number', + }, + { + name: 'file', + type: 'keyword', + description: 'file name', + }, + { + name: 'net', + type: 'keyword', + description: 'network MAC address', + }, + { + name: 'virt', + type: 'keyword', + description: 'kind of virtualization being referenced', + }, + { + name: 'cap_pp', + type: 'keyword', + description: 'process permitted capability map', + }, + { + name: 'old-range', + type: 'keyword', + description: 'present SELinux range', + }, + { + name: 'resrc', + type: 'keyword', + description: 'resource being assigned', + }, + { + name: 'new-range', + type: 'keyword', + description: 'new SELinux range', + }, + { + name: 'obj_gid', + type: 'keyword', + description: 'group ID of object', + }, + { + name: 'proto', + type: 'keyword', + description: 'network protocol', + }, + { + name: 'old-disk', + type: 'keyword', + description: 'disk being removed from vm', + }, + { + name: 'audit_failure', + type: 'keyword', + description: "audit system's failure mode", + }, + { + name: 'inif', + type: 'keyword', + description: 'in interface number', + }, + { + name: 'vm', + type: 'keyword', + description: 'virtual machine name', + }, + { + name: 'flags', + type: 'keyword', + description: 'mmap syscall flags', + }, + { + name: 'nlnk-fam', + type: 'keyword', + description: 'netlink protocol number', + }, + { + name: 'old-fs', + type: 'keyword', + description: 'file system being removed from vm', + }, + { + name: 'old-ses', + type: 'keyword', + description: 'previous ses value', + }, + { + name: 'seqno', + type: 'keyword', + description: 'sequence number', + }, + { + name: 'fver', + type: 'keyword', + description: 'file system capabilities version number', + }, + { + name: 'qbytes', + type: 'keyword', + description: 'ipc objects quantity of bytes', + }, + { + name: 'seuser', + type: 'keyword', + description: "user's SE Linux user acct", + }, + { + name: 'cap_fe', + type: 'keyword', + description: 'file assigned effective capability map', + }, + { + name: 'new-vcpu', + type: 'keyword', + description: 'new number of CPU cores', + }, + { + name: 'old-level', + type: 'keyword', + description: 'old run level', + }, + { + name: 'old_pp', + type: 'keyword', + description: 'old process permitted capability map', + }, + { + name: 'daddr', + type: 'keyword', + description: 'remote IP address', + }, + { + name: 'old-role', + type: 'keyword', + description: 'present SELinux role', + }, + { + name: 'ioctlcmd', + type: 'keyword', + description: 'The request argument to the ioctl syscall', + }, + { + name: 'smac', + type: 'keyword', + description: 'local MAC address', + }, + { + name: 'apparmor', + type: 'keyword', + description: 'apparmor event information', + }, + { + name: 'fe', + type: 'keyword', + description: 'file assigned effective capability map', + }, + { + name: 'perm_mask', + type: 'keyword', + description: 'file permission mask that triggered a watch event', + }, + { + name: 'ses', + type: 'keyword', + description: 'login session ID', + }, + { + name: 'cap_fi', + type: 'keyword', + description: 'file inherited capability map', + }, + { + name: 'obj_uid', + type: 'keyword', + description: 'user ID of object', + }, + { + name: 'reason', + type: 'keyword', + description: 'text string denoting a reason for the action', + }, + { + name: 'list', + type: 'keyword', + description: "the audit system's filter list number", + }, + { + name: 'old_lock', + type: 'keyword', + description: 'present value of feature lock', + }, + { + name: 'bus', + type: 'keyword', + description: 'name of subsystem bus a vm resource belongs to', + }, + { + name: 'old_pe', + type: 'keyword', + description: 'old process effective capability map', + }, + { + name: 'new-role', + type: 'keyword', + description: 'new SELinux role', + }, + { + name: 'prom', + type: 'keyword', + description: 'network promiscuity flag', + }, + { + name: 'uri', + type: 'keyword', + description: 'URI pointing to a printer', + }, + { + name: 'audit_enabled', + type: 'keyword', + description: "audit systems's enable/disable status", + }, + { + name: 'old-log_passwd', + type: 'keyword', + description: 'present value for TTY password logging', + }, + { + name: 'old-seuser', + type: 'keyword', + description: 'present SELinux user', + }, + { + name: 'per', + type: 'keyword', + description: 'linux personality', + }, + { + name: 'scontext', + type: 'keyword', + description: "the subject's context string", + }, + { + name: 'tclass', + type: 'keyword', + description: "target's object classification", + }, + { + name: 'ver', + type: 'keyword', + description: "audit daemon's version number", + }, + { + name: 'new', + type: 'keyword', + description: 'value being set in feature', + }, + { + name: 'val', + type: 'keyword', + description: 'generic value associated with the operation', + }, + { + name: 'img-ctx', + type: 'keyword', + description: "the vm's disk image context string", + }, + { + name: 'old-chardev', + type: 'keyword', + description: 'present character device assigned to vm', + }, + { + name: 'old_val', + type: 'keyword', + description: 'current value of SELinux boolean', + }, + { + name: 'success', + type: 'keyword', + description: 'whether the syscall was successful or not', + }, + { + name: 'inode_uid', + type: 'keyword', + description: "user ID of the inode's owner", + }, + { + name: 'removed', + type: 'keyword', + description: 'number of deleted files', + }, + { + name: 'socket', + type: 'group', + fields: [ + { + name: 'port', + type: 'keyword', + description: 'The port number.', + }, + { + name: 'saddr', + type: 'keyword', + description: 'The raw socket address structure.', + }, + { + name: 'addr', + type: 'keyword', + description: 'The remote address.', + }, + { + name: 'family', + type: 'keyword', + example: 'unix', + description: 'The socket family (unix, ipv4, ipv6, netlink).', + }, + { + name: 'path', + type: 'keyword', + description: 'This is the path associated with a unix socket.', + }, + ], + }, + ], }, { - name: 'update', + name: 'messages', + type: 'alias', + migration: true, + path: 'event.original', description: - 'A BSON document that specifies the update to be performed. For information on specifying updates, see the Update Operations documentation from the MongoDB Manual.', + 'An ordered list of the raw messages received from the kernel that were used to construct this document. This field is present if an error occurred processing the data or if `include_raw_message` is set in the config.\n', }, { - name: 'cursorId', + name: 'warnings', + type: 'alias', + migration: true, + path: 'error.message', description: - 'The cursor identifier returned in the OP_REPLY. This must be the value that was returned from the database.', + 'The warnings generated by the Beat during the construction of the event. These are disabled by default and are used for development and debug purposes only.\n', }, ], }, - ], - }, - { - key: 'mysql', - title: 'MySQL', - description: 'MySQL-specific event fields.', - fields: [ { - name: 'mysql', + name: 'geoip', type: 'group', + description: + 'The geoip fields are defined as a convenience in case you decide to enrich the data using a geoip filter in Logstash or Ingest Node.\n', fields: [ { - name: 'affected_rows', - type: 'long', - description: - 'If the MySQL command is successful, this field contains the affected number of rows of the last statement.', - }, - { - name: 'insert_id', - description: - 'If the INSERT query is successful, this field contains the id of the newly inserted row.', - }, - { - name: 'num_fields', - description: - 'If the SELECT query is successful, this field is set to the number of fields returned.', + name: 'continent_name', + type: 'keyword', + description: 'The name of the continent.\n', }, { - name: 'num_rows', - description: - 'If the SELECT query is successful, this field is set to the number of rows returned.', + name: 'city_name', + type: 'keyword', + description: 'The name of the city.\n', }, { - name: 'query', - description: "The row mysql query as read from the transaction's request. ", + name: 'region_name', + type: 'keyword', + description: 'The name of the region.\n', }, { - name: 'error_code', - type: 'long', - description: 'The error code returned by MySQL.', + name: 'country_iso_code', + type: 'keyword', + description: 'Country ISO code.\n', }, { - name: 'error_message', - description: 'The error info message returned by MySQL.', + name: 'location', + type: 'geo_point', + description: 'The longitude and latitude.\n', }, ], }, ], }, { - key: 'nfs', - title: 'NFS', - description: 'NFS v4/3 specific event fields.', + key: 'file_integrity', + title: 'File Integrity', + description: 'These are the fields generated by the file_integrity module.', fields: [ { - name: 'nfs', - type: 'group', - fields: [ - { - name: 'version', - type: 'long', - description: 'NFS protocol version number.', - }, - { - name: 'minor_version', - type: 'long', - description: 'NFS protocol minor version number.', - }, - { - name: 'tag', - description: 'NFS v4 COMPOUND operation tag.', - }, - { - name: 'opcode', - description: 'NFS operation name, or main operation name, in case of COMPOUND calls.', - }, - { - name: 'status', - description: 'NFS operation reply status.', - }, - ], - }, - { - name: 'rpc', + name: 'hash', type: 'group', - description: 'ONC RPC specific event fields.', + description: + 'Hashes of the file. The keys are algorithm names and the values are the hex encoded digest values.\n', fields: [ { - name: 'xid', - description: 'RPC message transaction identifier.', - }, - { - name: 'status', - description: 'RPC message reply status.', - }, - { - name: 'auth_flavor', - description: 'RPC authentication flavor.', - }, - { - name: 'cred.uid', - type: 'long', - description: "RPC caller's user id, in case of auth-unix.", - }, - { - name: 'cred.gid', - type: 'long', - description: "RPC caller's group id, in case of auth-unix.", - }, - { - name: 'cred.gids', - description: "RPC caller's secondary group ids, in case of auth-unix.", + name: 'blake2b_256', + type: 'keyword', + description: 'BLAKE2b-256 hash of the file.', }, { - name: 'cred.stamp', - type: 'long', - description: 'Arbitrary ID which the caller machine may generate.', + name: 'blake2b_384', + type: 'keyword', + description: 'BLAKE2b-384 hash of the file.', }, { - name: 'cred.machinename', - description: "The name of the caller's machine.", + name: 'blake2b_512', + type: 'keyword', + description: 'BLAKE2b-512 hash of the file.', }, { - name: 'call_size', - type: 'alias', - path: 'source.bytes', - migration: true, - description: 'RPC call size with argument.', + name: 'md5', + overwrite: true, + type: 'keyword', + description: 'MD5 hash of the file.', }, { - name: 'reply_size', - type: 'alias', - path: 'destination.bytes', - migration: true, - description: 'RPC reply size with argument.', + name: 'sha1', + overwrite: true, + type: 'keyword', + description: 'SHA1 hash of the file.', }, - ], - }, - ], - }, - { - key: 'pgsql', - title: 'PostgreSQL', - description: 'PostgreSQL-specific event fields.', - fields: [ - { - name: 'pgsql', - type: 'group', - fields: [ { - name: 'error_code', - description: 'The PostgreSQL error code.', - type: 'long', + name: 'sha224', + type: 'keyword', + description: 'SHA224 hash of the file.', }, { - name: 'error_message', - description: 'The PostgreSQL error message.', + name: 'sha256', + overwrite: true, + type: 'keyword', + description: 'SHA256 hash of the file.', }, { - name: 'error_severity', - description: 'The PostgreSQL error severity.', - possible_values: ['ERROR', 'FATAL', 'PANIC'], + name: 'sha384', + type: 'keyword', + description: 'SHA384 hash of the file.', }, { - name: 'num_fields', - description: - 'If the SELECT query if successful, this field is set to the number of fields returned.', + name: 'sha3_224', + type: 'keyword', + description: 'SHA3_224 hash of the file.', }, { - name: 'num_rows', - description: - 'If the SELECT query if successful, this field is set to the number of rows returned.', + name: 'sha3_256', + type: 'keyword', + description: 'SHA3_256 hash of the file.', }, - ], - }, - ], - }, - { - key: 'redis', - title: 'Redis', - description: 'Redis-specific event fields.', - fields: [ - { - name: 'redis', - type: 'group', - fields: [ { - name: 'return_value', - description: 'The return value of the Redis command in a human readable format.', + name: 'sha3_384', + type: 'keyword', + description: 'SHA3_384 hash of the file.', }, { - name: 'error', - description: - 'If the Redis command has resulted in an error, this field contains the error message returned by the Redis server.', + name: 'sha3_512', + type: 'keyword', + description: 'SHA3_512 hash of the file.', }, - ], - }, - ], - }, - { - key: 'thrift', - title: 'Thrift-RPC', - description: 'Thrift-RPC specific event fields.', - fields: [ - { - name: 'thrift', - type: 'group', - fields: [ { - name: 'params', - description: - 'The RPC method call parameters in a human readable format. If the IDL files are available, the parameters use names whenever possible. Otherwise, the IDs from the message are used.', + name: 'sha512', + overwrite: true, + type: 'keyword', + description: 'SHA512 hash of the file.', }, { - name: 'service', - description: 'The name of the Thrift-RPC service as defined in the IDL files.', + name: 'sha512_224', + type: 'keyword', + description: 'SHA512/224 hash of the file.', }, { - name: 'return_value', - description: - 'The value returned by the Thrift-RPC call. This is encoded in a human readable format.', + name: 'sha512_256', + type: 'keyword', + description: 'SHA512/256 hash of the file.', }, { - name: 'exceptions', - description: - 'If the call resulted in exceptions, this field contains the exceptions in a human readable format.', + name: 'xxh64', + type: 'keyword', + description: 'XX64 hash of the file.', }, ], }, ], }, { - key: 'tls', - title: 'TLS', - description: 'TLS-specific event fields.', + key: 'system', + title: 'System', + description: 'These are the fields generated by the system module.\n', + release: 'beta', fields: [ { - name: 'tls', + name: 'event', type: 'group', fields: [ { - name: 'version', + name: 'origin', type: 'keyword', - description: 'The version of the TLS protocol used.', - example: 'TLS 1.3', - }, - { - name: 'handshake_completed', - type: 'boolean', description: - 'Whether the TLS negotiation has been successful and the session has transitioned to encrypted mode.', - }, - { - name: 'resumed', - type: 'boolean', - description: 'If the TLS session has been resumed from a previous session.', + 'Origin of the event. This can be a file path (e.g. `/var/log/log.1`), or the name of the system component that supplied the data (e.g. `netlink`).\n', }, + ], + }, + { + name: 'user', + type: 'group', + fields: [ { - name: 'resumption_method', + name: 'entity_id', type: 'keyword', description: - 'If the session has been resumed, the underlying method used. One of "id" for TLS session ID or "ticket" for TLS ticket extension.', + 'ID uniquely identifying the user on a host. It is computed as a SHA-256 hash of the host ID, user ID, and user name.\n', }, { - name: 'client_certificate_requested', - type: 'boolean', - description: - 'Whether the server has requested the client to authenticate itself using a client certificate.', + name: 'terminal', + type: 'keyword', + description: 'Terminal of the user.\n', }, + ], + }, + { + name: 'process', + type: 'group', + fields: [ { - name: 'client_hello', + name: 'hash', type: 'group', + description: + 'Hashes of the executable. The keys are algorithm names and the values are the hex encoded digest values.\n', fields: [ { - name: 'version', + name: 'blake2b_256', type: 'keyword', - description: - 'The version of the TLS protocol by which the client wishes to communicate during this session.', + description: 'BLAKE2b-256 hash of the executable.', }, { - name: 'supported_ciphers', - type: 'array', - description: - 'List of ciphers the client is willing to use for this session. See https://www.iana.org/assignments/tls-parameters/tls-parameters.xhtml#tls-parameters-4', + name: 'blake2b_384', + type: 'keyword', + description: 'BLAKE2b-384 hash of the executable.', }, { - name: 'supported_compression_methods', - type: 'array', - description: - 'The list of compression methods the client supports. See https://www.iana.org/assignments/comp-meth-ids/comp-meth-ids.xhtml', + name: 'blake2b_512', + type: 'keyword', + description: 'BLAKE2b-512 hash of the executable.', }, { - name: 'extensions', - type: 'group', - description: 'The hello extensions provided by the client.', - fields: [ - { - name: 'server_name_indication', - type: 'keyword', - description: 'List of hostnames', - }, - { - name: 'application_layer_protocol_negotiation', - type: 'keyword', - description: - 'List of application-layer protocols the client is willing to use.', - }, - { - name: 'session_ticket', - type: 'keyword', - description: - 'Length of the session ticket, if provided, or an empty string to advertise support for tickets.', - }, - { - name: 'supported_versions', - type: 'keyword', - description: 'List of TLS versions that the client is willing to use.', - }, - { - name: 'supported_groups', - type: 'keyword', - description: - 'List of Elliptic Curve Cryptography (ECC) curve groups supported by the client.', - }, - { - name: 'signature_algorithms', - type: 'keyword', - description: - 'List of signature algorithms that may be use in digital signatures.', - }, - { - name: 'ec_points_formats', - type: 'keyword', - description: - 'List of Elliptic Curve (EC) point formats. Indicates the set of point formats that the client can parse.', - }, - { - name: '_unparsed_', - type: 'keyword', - description: 'List of extensions that were left unparsed by Packetbeat.', - }, - ], + name: 'sha224', + type: 'keyword', + description: 'SHA224 hash of the executable.', }, - ], - }, - { - name: 'server_hello', - type: 'group', - fields: [ { - name: 'version', + name: 'sha384', type: 'keyword', - description: - 'The version of the TLS protocol that is used for this session. It is the highest version supported by the server not exceeding the version requested in the client hello.', + description: 'SHA384 hash of the executable.', }, { - name: 'selected_cipher', + name: 'sha3_224', type: 'keyword', - description: - 'The cipher suite selected by the server from the list provided by in the client hello.', + description: 'SHA3_224 hash of the executable.', }, { - name: 'selected_compression_method', + name: 'sha3_256', type: 'keyword', - description: - 'The compression method selected by the server from the list provided in the client hello.', + description: 'SHA3_256 hash of the executable.', }, { - name: 'session_id', + name: 'sha3_384', type: 'keyword', - description: - 'Unique number to identify the session for the corresponding connection with the client.', + description: 'SHA3_384 hash of the executable.', }, { - name: 'extensions', - type: 'group', - description: 'The hello extensions provided by the server.', - fields: [ - { - name: 'application_layer_protocol_negotiation', - type: 'array', - description: 'Negotiated application layer protocol', - }, - { - name: 'session_ticket', - type: 'keyword', - description: - 'Used to announce that a session ticket will be provided by the server. Always an empty string.', - }, - { - name: 'supported_versions', - type: 'keyword', - description: 'Negotiated TLS version to be used.', - }, - { - name: 'ec_points_formats', - type: 'keyword', - description: - 'List of Elliptic Curve (EC) point formats. Indicates the set of point formats that the server can parse.', - }, - { - name: '_unparsed_', - type: 'keyword', - description: 'List of extensions that were left unparsed by Packetbeat.', - }, - ], + name: 'sha3_512', + type: 'keyword', + description: 'SHA3_512 hash of the executable.', + }, + { + name: 'sha512_224', + type: 'keyword', + description: 'SHA512/224 hash of the executable.', + }, + { + name: 'sha512_256', + type: 'keyword', + description: 'SHA512/256 hash of the executable.', + }, + { + name: 'xxh64', + type: 'keyword', + description: 'XX64 hash of the executable.', }, ], }, + ], + }, + { + name: 'socket', + type: 'group', + fields: [ + { + name: 'entity_id', + type: 'keyword', + description: + 'ID uniquely identifying the socket. It is computed as a SHA-256 hash of the host ID, socket inode, local IP, local port, remote IP, and remote port.\n', + }, + ], + }, + { + name: 'system.audit', + type: 'group', + description: '\n', + fields: [ { - name: 'client_certificate', + name: 'host', type: 'group', - description: 'Certificate provided by the client for authentication.', + description: '`host` contains general host information.\n', + release: 'beta', fields: [ { - name: 'version', + name: 'uptime', type: 'long', - description: 'X509 format version.', - }, - { - name: 'serial_number', - type: 'keyword', - description: "The certificate's serial number.", + format: 'duration', + input_format: 'nanoseconds', + output_format: 'asDays', + output_precision: 1, + description: 'Uptime in nanoseconds.\n', }, { - name: 'not_before', + name: 'boottime', type: 'date', - description: 'Date before which the certificate is not valid.', + description: 'Boot time.\n', }, { - name: 'not_after', - type: 'date', - description: 'Date after which the certificate expires.', + name: 'containerized', + type: 'boolean', + description: 'Set if host is a container.\n', }, { - name: 'public_key_algorithm', + name: 'timezone.name', type: 'keyword', - description: - "The algorithm used for this certificate's public key. One of RSA, DSA or ECDSA. ", + description: 'Name of the timezone of the host, e.g. BST.\n', }, { - name: 'public_key_size', + name: 'timezone.offset.sec', type: 'long', - description: 'Size of the public key.', + description: 'Timezone offset in seconds.\n', + }, + { + name: 'hostname', + type: 'keyword', + description: 'Hostname.\n', }, { - name: 'signature_algorithm', + name: 'id', type: 'keyword', - description: "The algorithm used for the certificate's signature. ", + description: 'Host ID.\n', }, { - name: 'alternative_names', - type: 'array', - description: 'Subject Alternative Names for this certificate.', + name: 'architecture', + type: 'keyword', + description: 'Host architecture (e.g. x86_64).\n', }, { - name: 'raw', + name: 'mac', type: 'keyword', - description: 'The raw certificate in PEM format.', + description: 'MAC addresses.\n', }, { - name: 'subject', - type: 'group', - description: 'Subject represented by this certificate.', - fields: [ - { - name: 'country', - type: 'keyword', - description: 'Country code.', - }, - { - name: 'organization', - type: 'keyword', - description: 'Organization name.', - }, - { - name: 'organizational_unit', - type: 'keyword', - description: 'Unit within organization.', - }, - { - name: 'province', - type: 'keyword', - description: 'Province or region within country.', - }, - { - name: 'common_name', - type: 'keyword', - description: 'Name or host name identified by the certificate.', - }, - ], + name: 'ip', + type: 'ip', + description: 'IP addresses.\n', }, { - name: 'issuer', + name: 'os', type: 'group', - description: 'Entity that issued and signed this certificate.', + description: '`os` contains information about the operating system.\n', fields: [ { - name: 'country', + name: 'codename', type: 'keyword', - description: 'Country code.', + description: 'OS codename, if any (e.g. stretch).\n', }, { - name: 'organization', + name: 'platform', type: 'keyword', - description: 'Organization name.', + description: 'OS platform (e.g. centos, ubuntu, windows).\n', }, { - name: 'organizational_unit', + name: 'name', type: 'keyword', - description: 'Unit within organization.', + description: 'OS name (e.g. Mac OS X).\n', }, { - name: 'province', - type: 'keyword', - description: 'Province or region within country.', - }, - { - name: 'common_name', - type: 'keyword', - description: 'Name or host name identified by the certificate.', - }, - ], - }, - { - name: 'fingerprint', - type: 'group', - fields: [ - { - name: 'md5', + name: 'family', type: 'keyword', - description: "Certificate's MD5 fingerprint.", + description: 'OS family (e.g. redhat, debian, freebsd, windows).\n', }, { - name: 'sha1', + name: 'version', type: 'keyword', - description: "Certificate's SHA-1 fingerprint.", + description: 'OS version.\n', }, { - name: 'sha256', + name: 'kernel', type: 'keyword', - description: "Certificate's SHA-256 fingerprint.", + description: "The operating system's kernel version.\n", }, ], }, ], }, { - name: 'server_certificate', + name: 'package', type: 'group', - description: 'Certificate provided by the server for authentication.', + description: '`package` contains information about an installed or removed package.\n', + release: 'beta', fields: [ + { + name: 'entity_id', + type: 'keyword', + description: + 'ID uniquely identifying the package. It is computed as a SHA-256 hash of the host ID, package name, and package version.\n', + }, + { + name: 'name', + type: 'keyword', + description: 'Package name.\n', + }, { name: 'version', - type: 'long', - description: 'X509 format version.', + type: 'keyword', + description: 'Package version.\n', }, { - name: 'serial_number', + name: 'release', type: 'keyword', - description: "The certificate's serial number.", + description: 'Package release.\n', }, { - name: 'not_before', - type: 'date', - description: 'Date before which the certificate is not valid.', + name: 'arch', + type: 'keyword', + description: 'Package architecture.\n', }, { - name: 'not_after', + name: 'license', + type: 'keyword', + description: 'Package license.\n', + }, + { + name: 'installtime', type: 'date', - description: 'Date after which the certificate expires.', + description: 'Package install time.\n', + }, + { + name: 'size', + type: 'long', + description: 'Package size.\n', + }, + { + name: 'summary', + description: 'Package summary.\n', }, { - name: 'public_key_algorithm', + name: 'url', type: 'keyword', - description: - "The algorithm used for this certificate's public key. One of RSA, DSA or ECDSA. ", + description: 'Package URL.\n', }, + ], + }, + { + name: 'user', + type: 'group', + description: '`user` contains information about the users on a system.\n', + release: 'beta', + fields: [ { - name: 'public_key_size', - type: 'long', - description: 'Size of the public key.', + name: 'name', + type: 'keyword', + description: 'User name.\n', }, { - name: 'signature_algorithm', + name: 'uid', type: 'keyword', - description: "The algorithm used for the certificate's signature. ", + description: 'User ID.\n', }, { - name: 'alternative_names', - type: 'array', - description: 'Subject Alternative Names for this certificate.', + name: 'gid', + type: 'keyword', + description: 'Group ID.\n', }, { - name: 'raw', + name: 'dir', type: 'keyword', - description: 'The raw certificate in PEM format.', + description: "User's home directory.\n", }, { - name: 'subject', - type: 'group', - description: 'Subject represented by this certificate.', - fields: [ - { - name: 'country', - type: 'keyword', - description: 'Country code.', - }, - { - name: 'organization', - type: 'keyword', - description: 'Organization name.', - }, - { - name: 'organizational_unit', - type: 'keyword', - description: 'Unit within organization.', - }, - { - name: 'province', - type: 'keyword', - description: 'Province or region within country.', - }, - { - name: 'common_name', - type: 'keyword', - description: 'Name or host name identified by the certificate.', - }, - ], + name: 'shell', + type: 'keyword', + description: 'Program to run at login.\n', }, { - name: 'issuer', - type: 'group', - description: 'Entity that issued and signed this certificate.', - fields: [ - { - name: 'country', - type: 'keyword', - description: 'Country code.', - }, - { - name: 'organization', - type: 'keyword', - description: 'Organization name.', - }, - { - name: 'organizational_unit', - type: 'keyword', - description: 'Unit within organization.', - }, - { - name: 'province', - type: 'keyword', - description: 'Province or region within country.', - }, - { - name: 'common_name', - type: 'keyword', - description: 'Name or host name identified by the certificate.', - }, - ], + name: 'user_information', + type: 'keyword', + description: 'General user information. On Linux, this is the gecos field.\n', }, { - name: 'fingerprint', - type: 'group', + name: 'group', + type: 'object', + description: + "`group` contains information about any groups the user is part of (beyond the user's primary group).\n", fields: [ { - name: 'md5', + name: 'name', type: 'keyword', - description: "Certificate's MD5 fingerprint.", + description: 'Group name.\n', }, { - name: 'sha1', - type: 'keyword', - description: "Certificate's SHA-1 fingerprint.", - }, - { - name: 'sha256', - type: 'keyword', - description: "Certificate's SHA-256 fingerprint.", + name: 'gid', + type: 'integer', + description: 'Group ID.\n', }, ], }, - ], - }, - { - name: 'server_certificate_chain', - type: 'array', - description: 'Chain of trust for the server certificate.', - }, - { - name: 'client_certificate_chain', - type: 'array', - description: 'Chain of trust for the client certificate.', - }, - { - name: 'alert_types', - type: 'keyword', - description: 'An array containing the TLS alert type for every alert received.', - }, - { - name: 'fingerprints', - type: 'group', - description: 'Fingerprints for this TLS session.', - fields: [ { - name: 'ja3', + name: 'password', type: 'group', - description: 'JA3 TLS client fingerprint', + description: + "`password` contains information about a user's password (not the password itself).\n", fields: [ { - name: 'hash', + name: 'type', type: 'keyword', - description: 'The JA3 fingerprint hash for the client side.', + description: + "A user's password type. Possible values are `shadow_password` (the password hash is in the shadow file), `password_disabled`, `no_password` (this is dangerous as anyone can log in), and `crypt_password` (when the password field in /etc/passwd seems to contain an encrypted password).\n", }, { - name: 'str', - type: 'keyword', - description: 'The JA3 string used to calculate the hash.', + name: 'last_changed', + type: 'date', + description: "The day the user's password was last changed.\n", }, ], }, diff --git a/x-pack/legacy/plugins/siem/server/utils/beat_schema/8.0.0/ecs.ts b/x-pack/legacy/plugins/siem/server/utils/beat_schema/8.0.0/ecs.ts index 34deee7fa8895..a439d105d63df 100644 --- a/x-pack/legacy/plugins/siem/server/utils/beat_schema/8.0.0/ecs.ts +++ b/x-pack/legacy/plugins/siem/server/utils/beat_schema/8.0.0/ecs.ts @@ -13,1997 +13,5663 @@ * instances e.g. `@timestamp` to `timestamp` */ -import { EcsSchema } from '../type'; +import { Schema } from '../type'; -export const ecsSchema: EcsSchema = { - agent: { - description: - 'The agent fields contain the data about the software entity, if any, that collects, detects, or observes events on a host, or takes measurements on a host. Examples include Beats. Agents may also run on observers. ECS agent.* fields shall be populated with details of the agent running on the host or observer where the event happened or the measurement was taken.\n', - fields: { - 'agent.ephemeral_id': { - description: - 'Ephemeral identifier of this agent (if one exists).\nThis id normally changes across restarts, but `agent.id` does not.', - example: '8a4f500f', - footnote: '', - group: 2, - level: 'extended', - name: 'agent.ephemeral_id', - required: false, - type: 'keyword', - }, - 'agent.id': { - description: - 'Unique identifier of this agent (if one exists).\nExample: For Beats this would be beat.id.', - example: '8a4f500d', - footnote: '', - group: 2, - level: 'core', - name: 'agent.id', - required: false, - type: 'keyword', - }, - 'agent.name': { - description: - 'Name of the agent.\nThis is a name that can be given to an agent. This can be helpful if for example two Filebeat instances are running on the same host but a human readable separation is needed on which Filebeat instance data is coming from.\nIf no name is given, the name is often left empty.', - example: 'foo', - footnote: '', - group: 2, - level: 'core', - name: 'agent.name', - required: false, - type: 'keyword', - }, - 'agent.type': { - description: - 'Type of the agent.\nThe agent type stays always the same and should be given by the agent used. In case of Filebeat the agent would always be Filebeat also if two Filebeat instances are run on the same machine.', - example: 'filebeat', - footnote: '', - group: 2, - level: 'core', - name: 'agent.type', - required: false, - type: 'keyword', - }, - 'agent.version': { - description: 'Version of the agent.', - example: '6.0.0-rc2', - footnote: '', - group: 2, - level: 'core', - name: 'agent.version', - required: false, - type: 'keyword', - }, - }, - group: 2, - name: 'agent', - title: 'Agent', - type: 'group', - }, - base: { - description: - 'The base set contains all fields which are on the top level. These fields are common across all types of events.\n', - fields: { - '@timestamp': { - description: - 'Date/time when the event originated.\nFor log events this is the date/time when the event was generated, and not when it was read.\nRequired field for all events.', - example: '2016-05-23T08:05:34.853Z', - footnote: '', - group: 1, - level: 'core', +export const ecsSchema: Schema = [ + { + key: 'ecs', + title: 'ECS', + description: 'ECS Fields.', + fields: [ + { name: '@timestamp', + level: 'core', required: true, type: 'date', - }, - labels: { description: - 'Key/value pairs.\nCan be used to add meta information to events. Should not contain nested objects. All values are stored as keyword.\nExample: `docker` and `k8s` labels.', - example: "{'application': 'foo-bar', 'env': 'production'}", - footnote: '', - group: 1, - level: 'core', + 'Date/time when the event originated.\n\nThis is the date/time extracted from the event, typically representing when\nthe event was generated by the source.\n\nIf the event source has no original timestamp, this value is typically populated\nby the first time the event was received by the pipeline.\n\nRequired field for all events.', + example: '2016-05-23T08:05:34.853Z', + }, + { name: 'labels', - required: false, + level: 'core', type: 'object', - }, - message: { + object_type: 'keyword', description: - 'For log events the message field contains the log message.\nIn other use cases the message field can be used to concatenate different values which are then freely searchable. If multiple messages exist, they can be combined into one message.', - example: 'Hello World', - footnote: '', - group: 1, - level: 'core', + 'Custom key/value pairs.\n\nCan be used to add meta information to events. Should not contain nested objects.\nAll values are stored as keyword.\n\nExample: `docker` and `k8s` labels.', + example: '{"application": "foo-bar", "env": "production"}', + }, + { name: 'message', - required: false, + level: 'core', type: 'text', + description: + 'For log events the message field contains the log message, optimized\nfor viewing in a log viewer.\n\nFor structured logs without an original message field, other fields can be concatenated\nto form a human-readable summary of the event.\n\nIf multiple messages exist, they can be combined into one message.', + example: 'Hello World', }, - tags: { - description: 'List of keywords used to tag each event.', - example: '["production", "env2"]', - footnote: '', - group: 1, - level: 'core', + { name: 'tags', - required: false, - type: 'keyword', - }, - }, - group: 1, - name: 'base', - title: 'Base', - type: 'group', - }, - client: { - description: - 'A client is defined as the initiator of a network connection for events regarding sessions, connections, or bidirectional flow records. For TCP events, the client is the initiator of the TCP connection that sends the SYN packet(s). For other protocols, the client is generally the initiator or requestor in the network transaction. Some systems use the term "originator" to refer the client in TCP connections. The client fields describe details about the system acting as the client in the network event. Client fields are usually populated in conjuction with server fields. Client fields are generally not populated for packet-level events. \n', - fields: { - 'client.bytes': { - description: 'Bytes sent from the client to the server.', - example: '184', - footnote: '', - group: 2, level: 'core', - name: 'client.bytes', - required: false, - type: 'long', - }, - 'client.domain': { - description: 'Client domain.', - example: '', - footnote: '', - group: 2, - level: 'core', - name: 'client.domain', - required: false, - type: 'keyword', - }, - 'client.ip': { - description: 'IP address of the client.\nCan be one or multiple IPv4 or IPv6 addresses.', - example: '', - footnote: '', - group: 2, - level: 'core', - name: 'client.ip', - required: false, - type: 'ip', - }, - 'client.mac': { - description: 'MAC address of the client.', - example: '', - footnote: '', - group: 2, - level: 'core', - name: 'client.mac', - required: false, type: 'keyword', + ignore_above: 1024, + description: 'List of keywords used to tag each event.', + example: '["production", "env2"]', }, - 'client.packets': { - description: 'Packets sent from the client to the server.', - example: '12', - footnote: '', - group: 2, - level: 'core', - name: 'client.packets', - required: false, - type: 'long', - }, - 'client.port': { - description: 'Port of the client.', - example: '', - footnote: '', + { + name: 'agent', + title: 'Agent', group: 2, - level: 'core', - name: 'client.port', - required: false, - type: 'long', - }, - }, - group: 2, - name: 'client', - title: 'Client', - type: 'group', - }, - cloud: { - description: 'Fields related to the cloud or infrastructure the events are coming from.\n', - fields: { - 'cloud.account.id': { description: - 'The cloud account or organization id used to identify different entities in a multi-tenant environment.\nExamples: AWS account id, Google Cloud ORG Id, or other unique identifier.', - example: '666777888999', - footnote: '', - group: 2, - level: 'extended', - name: 'cloud.account.id', - required: false, - type: 'keyword', - }, - 'cloud.availability_zone': { - description: 'Availability zone in which this host is running.', - example: 'us-east-1c', - footnote: '', - group: 2, - level: 'extended', - name: 'cloud.availability_zone', - required: false, - type: 'keyword', - }, - 'cloud.instance.id': { - description: 'Instance ID of the host machine.', - example: 'i-1234567890abcdef0', - footnote: '', - group: 2, - level: 'extended', - name: 'cloud.instance.id', - required: false, - type: 'keyword', - }, - 'cloud.instance.name': { - description: 'Instance name of the host machine.', - example: '', - footnote: '', - group: 2, - level: 'extended', - name: 'cloud.instance.name', - required: false, - type: 'keyword', - }, - 'cloud.machine.type': { - description: 'Machine type of the host machine.', - example: 't2.medium', - footnote: '', - group: 2, - level: 'extended', - name: 'cloud.machine.type', - required: false, - type: 'keyword', - }, - 'cloud.provider': { - description: 'Name of the cloud provider. Example values are ec2, gce, or digitalocean.', - example: 'ec2', - footnote: '', - group: 2, - level: 'extended', - name: 'cloud.provider', - required: false, - type: 'keyword', - }, - 'cloud.region': { - description: 'Region in which this host is running.', - example: 'us-east-1', - footnote: '', - group: 2, - level: 'extended', - name: 'cloud.region', - required: false, - type: 'keyword', - }, - }, - group: 2, - name: 'cloud', - title: 'Cloud', - type: 'group', - }, - container: { - description: - 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.\n', - fields: { - 'container.id': { - description: 'Unique container id.', - example: '', - footnote: '', - group: 2, - level: 'core', - name: 'container.id', - required: false, - type: 'keyword', - }, - 'container.image.name': { - description: 'Name of the image the container was built on.', - example: '', - footnote: '', - group: 2, - level: 'extended', - name: 'container.image.name', - required: false, - type: 'keyword', - }, - 'container.image.tag': { - description: 'Container image tag.', - example: '', - footnote: '', - group: 2, - level: 'extended', - name: 'container.image.tag', - required: false, - type: 'keyword', - }, - 'container.labels': { - description: 'Image labels.', - example: '', - footnote: '', - group: 2, - level: 'extended', - name: 'container.labels', - required: false, - type: 'object', - }, - 'container.name': { - description: 'Container name.', - example: '', - footnote: '', - group: 2, - level: 'extended', - name: 'container.name', - required: false, - type: 'keyword', - }, - 'container.runtime': { - description: 'Runtime managing this container.', - example: 'docker', - footnote: '', + 'The agent fields contain the data about the software entity, if\nany, that collects, detects, or observes events on a host, or takes measurements\non a host.\n\nExamples include Beats. Agents may also run on observers. ECS agent.* fields\nshall be populated with details of the agent running on the host or observer\nwhere the event happened or the measurement was taken.', + footnote: + 'Examples: In the case of Beats for logs, the agent.name is filebeat.\nFor APM, it is the agent running in the app/service. The agent information does\nnot change if data is sent through queuing systems like Kafka, Redis, or processing\nsystems such as Logstash or APM Server.', + type: 'group', + fields: [ + { + name: 'build.original', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'Extended build information for the agent.\n\nThis field is intended to contain any build information that a data source\nmay provide, no specific formatting is required.', + example: + 'metricbeat version 7.6.0 (amd64), libbeat 7.6.0 [6a23e8f8f30f5001ba344e4e54d8d9cb82cb107c\nbuilt 2020-02-05 23:10:10 +0000 UTC]', + default_field: false, + }, + { + name: 'ephemeral_id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Ephemeral identifier of this agent (if one exists).\n\nThis id normally changes across restarts, but `agent.id` does not.', + example: '8a4f500f', + }, + { + name: 'id', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'Unique identifier of this agent (if one exists).\n\nExample: For Beats this would be beat.id.', + example: '8a4f500d', + }, + { + name: 'name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'Custom name of the agent.\n\nThis is a name that can be given to an agent. This can be helpful if for example\ntwo Filebeat instances are running on the same host but a human readable separation\nis needed on which Filebeat instance data is coming from.\n\nIf no name is given, the name is often left empty.', + example: 'foo', + }, + { + name: 'type', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'Type of the agent.\n\nThe agent type stays always the same and should be given by the agent used.\nIn case of Filebeat the agent would always be Filebeat also if two Filebeat\ninstances are run on the same machine.', + example: 'filebeat', + }, + { + name: 'version', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Version of the agent.', + example: '6.0.0-rc2', + }, + ], + }, + { + name: 'as', + title: 'Autonomous System', group: 2, - level: 'extended', - name: 'container.runtime', - required: false, - type: 'keyword', - }, - }, - group: 2, - name: 'container', - title: 'Container', - type: 'group', - }, - destination: { - description: - 'Destination fields describe details about the destination of a packet/event. Destination fields are usually populated in conjunction with source fields.\n', - fields: { - 'destination.bytes': { - description: 'Bytes sent from the destination to the source.', - example: '184', - footnote: '', - group: 2, - level: 'core', - name: 'destination.bytes', - required: false, - type: 'long', - }, - 'destination.domain': { - description: 'Destination domain.', - example: '', - footnote: '', - group: 2, - level: 'core', - name: 'destination.domain', - required: false, - type: 'keyword', - }, - 'destination.ip': { description: - 'IP address of the destination.\nCan be one or multiple IPv4 or IPv6 addresses.', - example: '', - footnote: '', - group: 2, - level: 'core', - name: 'destination.ip', - required: false, - type: 'ip', - }, - 'destination.mac': { - description: 'MAC address of the destination.', - example: '', - footnote: '', - group: 2, - level: 'core', - name: 'destination.mac', - required: false, - type: 'keyword', - }, - 'destination.packets': { - description: 'Packets sent from the destination to the source.', - example: '12', - footnote: '', - group: 2, - level: 'core', - name: 'destination.packets', - required: false, - type: 'long', - }, - 'destination.port': { - description: 'Port of the destination.', - example: '', - footnote: '', + 'An autonomous system (AS) is a collection of connected Internet Protocol\n(IP) routing prefixes under the control of one or more network operators on\nbehalf of a single administrative entity or domain that presents a common, clearly\ndefined routing policy to the internet.', + type: 'group', + fields: [ + { + name: 'number', + level: 'extended', + type: 'long', + description: + 'Unique number allocated to the autonomous system. The autonomous\nsystem number (ASN) uniquely identifies each network on the Internet.', + example: 15169, + }, + { + name: 'organization.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Organization name.', + example: 'Google LLC', + }, + ], + }, + { + name: 'client', + title: 'Client', group: 2, - level: 'core', - name: 'destination.port', - required: false, - type: 'long', - }, - }, - group: 2, - name: 'destination', - title: 'Destination', - type: 'group', - }, - ecs: { - description: 'Meta-information specific to ECS.\n', - fields: { - 'ecs.version': { description: - 'ECS version this event conforms to. `ecs.version` is a required field and must exist in all events.\nWhen querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events.\nThe current version is 1.0.0-beta1 .', - example: '1.0.0-beta1', - footnote: '', - group: 2, - level: 'core', - name: 'ecs.version', - required: true, - type: 'keyword', - }, - }, - group: 2, - name: 'ecs', - title: 'ECS', - type: 'group', - }, - error: { - description: - 'These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error.\n', - fields: { - 'error.code': { - description: 'Error code describing the error.', - example: '', - footnote: '', - group: 2, - level: 'core', - name: 'error.code', - required: false, - type: 'keyword', - }, - 'error.id': { - description: 'Unique identifier for the error.', - example: '', - footnote: '', - group: 2, - level: 'core', - name: 'error.id', - required: false, - type: 'keyword', - }, - 'error.message': { - description: 'Error message.', - example: '', - footnote: '', + 'A client is defined as the initiator of a network connection for\nevents regarding sessions, connections, or bidirectional flow records.\n\nFor TCP events, the client is the initiator of the TCP connection that sends\nthe SYN packet(s). For other protocols, the client is generally the initiator\nor requestor in the network transaction. Some systems use the term "originator"\nto refer the client in TCP connections. The client fields describe details about\nthe system acting as the client in the network event. Client fields are usually\npopulated in conjunction with server fields. Client fields are generally not\npopulated for packet-level events.\n\nClient / server representations can add semantic context to an exchange, which\nis helpful to visualize the data in certain situations. If your context falls\nin that category, you should still ensure that source and destination are filled\nappropriately.', + type: 'group', + fields: [ + { + name: 'address', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Some event client addresses are defined ambiguously. The event\nwill sometimes list an IP, a domain or a unix socket. You should always store\nthe raw address in the `.address` field.\n\nThen it should be duplicated to `.ip` or `.domain`, depending on which one\nit is.', + }, + { + name: 'as.number', + level: 'extended', + type: 'long', + description: + 'Unique number allocated to the autonomous system. The autonomous\nsystem number (ASN) uniquely identifies each network on the Internet.', + example: 15169, + }, + { + name: 'as.organization.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Organization name.', + example: 'Google LLC', + }, + { + name: 'bytes', + level: 'core', + type: 'long', + format: 'bytes', + description: 'Bytes sent from the client to the server.', + example: 184, + }, + { + name: 'domain', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Client domain.', + }, + { + name: 'geo.city_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'City name.', + example: 'Montreal', + }, + { + name: 'geo.continent_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the continent.', + example: 'North America', + }, + { + name: 'geo.country_iso_code', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Country ISO code.', + example: 'CA', + }, + { + name: 'geo.country_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Country name.', + example: 'Canada', + }, + { + name: 'geo.location', + level: 'core', + type: 'geo_point', + description: 'Longitude and latitude.', + example: '{ "lon": -73.614830, "lat": 45.505918 }', + }, + { + name: 'geo.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'User-defined description of a location, at the level of granularity\nthey care about.\n\nCould be the name of their data centers, the floor number, if this describes\na local physical entity, city names.\n\nNot typically used in automated geolocation.', + example: 'boston-dc', + }, + { + name: 'geo.region_iso_code', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Region ISO code.', + example: 'CA-QC', + }, + { + name: 'geo.region_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Region name.', + example: 'Quebec', + }, + { + name: 'ip', + level: 'core', + type: 'ip', + description: 'IP address of the client (IPv4 or IPv6).', + }, + { + name: 'mac', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'MAC address of the client.', + }, + { + name: 'nat.ip', + level: 'extended', + type: 'ip', + description: + 'Translated IP of source based NAT sessions (e.g. internal client\nto internet).\n\nTypically connections traversing load balancers, firewalls, or routers.', + }, + { + name: 'nat.port', + level: 'extended', + type: 'long', + format: 'string', + description: + 'Translated port of source based NAT sessions (e.g. internal client\nto internet).\n\nTypically connections traversing load balancers, firewalls, or routers.', + }, + { + name: 'packets', + level: 'core', + type: 'long', + description: 'Packets sent from the client to the server.', + example: 12, + }, + { + name: 'port', + level: 'core', + type: 'long', + format: 'string', + description: 'Port of the client.', + }, + { + name: 'registered_domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The highest registered client domain, stripped of the subdomain.\n\nFor example, the registered domain for "foo.google.com" is "google.com".\n\nThis value can be determined precisely with a list like the public suffix\nlist (http://publicsuffix.org). Trying to approximate this by simply taking\nthe last two labels will not work well for TLDs such as "co.uk".', + example: 'google.com', + }, + { + name: 'top_level_domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The effective top level domain (eTLD), also known as the domain\nsuffix, is the last part of the domain name. For example, the top level domain\nfor google.com is "com".\n\nThis value can be determined precisely with a list like the public suffix\nlist (http://publicsuffix.org). Trying to approximate this by simply taking\nthe last label will not work well for effective TLDs such as "co.uk".', + example: 'co.uk', + }, + { + name: 'user.domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the directory the user is a member of.\n\nFor example, an LDAP or Active Directory domain name.', + }, + { + name: 'user.email', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'User email address.', + }, + { + name: 'user.full_name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: "User's full name, if available.", + example: 'Albert Einstein', + }, + { + name: 'user.group.domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the directory the group is a member of.\n\nFor example, an LDAP or Active Directory domain name.', + }, + { + name: 'user.group.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifier for the group on the system/platform.', + }, + { + name: 'user.group.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the group.', + }, + { + name: 'user.hash', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Unique user hash to correlate information for a user in anonymized\nform.\n\nUseful if `user.id` or `user.name` contain confidential information and cannot\nbe used.', + }, + { + name: 'user.id', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifier of the user.', + }, + { + name: 'user.name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Short name or login of the user.', + example: 'albert', + }, + ], + }, + { + name: 'cloud', + title: 'Cloud', + group: 2, + description: 'Fields related to the cloud or infrastructure the events are coming\nfrom.', + footnote: + 'Examples: If Metricbeat is running on an EC2 host and fetches data\nfrom its host, the cloud info contains the data about this machine. If Metricbeat\nruns on a remote machine outside the cloud and fetches data from a service running\nin the cloud, the field contains cloud data from the machine the service is\nrunning on.', + type: 'group', + fields: [ + { + name: 'account.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The cloud account or organization id used to identify different\nentities in a multi-tenant environment.\n\nExamples: AWS account id, Google Cloud ORG Id, or other unique identifier.', + example: 666777888999, + }, + { + name: 'availability_zone', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Availability zone in which this host is running.', + example: 'us-east-1c', + }, + { + name: 'instance.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Instance ID of the host machine.', + example: 'i-1234567890abcdef0', + }, + { + name: 'instance.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Instance name of the host machine.', + }, + { + name: 'machine.type', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Machine type of the host machine.', + example: 't2.medium', + }, + { + name: 'provider', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the cloud provider. Example values are aws, azure, gcp,\nor digitalocean.', + example: 'aws', + }, + { + name: 'region', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Region in which this host is running.', + example: 'us-east-1', + }, + ], + }, + { + name: 'code_signature', + title: 'Code Signature', + group: 2, + description: 'These fields contain information about binary code signatures.', + type: 'group', + fields: [ + { + name: 'exists', + level: 'core', + type: 'boolean', + description: 'Boolean to capture if a signature is present.', + example: 'true', + default_field: false, + }, + { + name: 'status', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Additional information about the certificate status.\n\nThis is useful for logging cryptographic errors with the certificate validity\nor trust status. Leave unpopulated if the validity or trust of the certificate\nwas unchecked.', + example: 'ERROR_UNTRUSTED_ROOT', + default_field: false, + }, + { + name: 'subject_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Subject name of the code signer', + example: 'Microsoft Corporation', + default_field: false, + }, + { + name: 'trusted', + level: 'extended', + type: 'boolean', + description: + 'Stores the trust status of the certificate chain.\n\nValidating the trust of the certificate chain may be complicated, and this\nfield should only be populated by tools that actively check the status.', + example: 'true', + default_field: false, + }, + { + name: 'valid', + level: 'extended', + type: 'boolean', + description: + 'Boolean to capture if the digital signature is verified against\nthe binary content.\n\nLeave unpopulated if a certificate was unchecked.', + example: 'true', + default_field: false, + }, + ], + }, + { + name: 'container', + title: 'Container', group: 2, - level: 'core', - name: 'error.message', - required: false, - type: 'text', - }, - }, - group: 2, - name: 'error', - title: 'Error', - type: 'group', - }, - event: { - description: - 'The event fields are used for context information about the log or metric event itself. A log is defined as an event containing details of something that happened. Log events must include the time at which the thing happened. Examples of log events include a process starting on a host, a network packet being sent from a source to a destination, or a network connection between a client and a server being initiated or closed. A metric is defined as an event containing one or more numerical or categorical measurements and the time at which the measurement was taken. Examples of metric events include memory pressure measured on a host, or vulnerabilities measured on a scanned host.\n', - fields: { - 'event.action': { description: - 'The action captured by the event.\nThis describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer.', - example: 'user-password-change', - footnote: '', + 'Container fields are used for meta information about the specific\ncontainer that is the source of information.\n\nThese fields help correlate data based containers from any runtime.', + type: 'group', + fields: [ + { + name: 'id', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Unique container id.', + }, + { + name: 'image.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the image the container was built on.', + }, + { + name: 'image.tag', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Container image tags.', + }, + { + name: 'labels', + level: 'extended', + type: 'object', + object_type: 'keyword', + description: 'Image labels.', + }, + { + name: 'name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Container name.', + }, + { + name: 'runtime', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Runtime managing this container.', + example: 'docker', + }, + ], + }, + { + name: 'destination', + title: 'Destination', group: 2, - level: 'core', - name: 'event.action', - required: false, - type: 'keyword', - }, - 'event.category': { description: - 'Event category.\nThis contains high-level information about the contents of the event. It is more generic than `event.action`, in the sense that typically a category contains multiple actions. Warning: In future versions of ECS, we plan to provide a list of acceptable values for this field, please use with caution.', - example: 'user-management', - footnote: '', + 'Destination fields describe details about the destination of a packet/event.\n\nDestination fields are usually populated in conjunction with source fields.', + type: 'group', + fields: [ + { + name: 'address', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Some event destination addresses are defined ambiguously. The\nevent will sometimes list an IP, a domain or a unix socket. You should always\nstore the raw address in the `.address` field.\n\nThen it should be duplicated to `.ip` or `.domain`, depending on which one\nit is.', + }, + { + name: 'as.number', + level: 'extended', + type: 'long', + description: + 'Unique number allocated to the autonomous system. The autonomous\nsystem number (ASN) uniquely identifies each network on the Internet.', + example: 15169, + }, + { + name: 'as.organization.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Organization name.', + example: 'Google LLC', + }, + { + name: 'bytes', + level: 'core', + type: 'long', + format: 'bytes', + description: 'Bytes sent from the destination to the source.', + example: 184, + }, + { + name: 'domain', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Destination domain.', + }, + { + name: 'geo.city_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'City name.', + example: 'Montreal', + }, + { + name: 'geo.continent_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the continent.', + example: 'North America', + }, + { + name: 'geo.country_iso_code', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Country ISO code.', + example: 'CA', + }, + { + name: 'geo.country_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Country name.', + example: 'Canada', + }, + { + name: 'geo.location', + level: 'core', + type: 'geo_point', + description: 'Longitude and latitude.', + example: '{ "lon": -73.614830, "lat": 45.505918 }', + }, + { + name: 'geo.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'User-defined description of a location, at the level of granularity\nthey care about.\n\nCould be the name of their data centers, the floor number, if this describes\na local physical entity, city names.\n\nNot typically used in automated geolocation.', + example: 'boston-dc', + }, + { + name: 'geo.region_iso_code', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Region ISO code.', + example: 'CA-QC', + }, + { + name: 'geo.region_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Region name.', + example: 'Quebec', + }, + { + name: 'ip', + level: 'core', + type: 'ip', + description: 'IP address of the destination (IPv4 or IPv6).', + }, + { + name: 'mac', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'MAC address of the destination.', + }, + { + name: 'nat.ip', + level: 'extended', + type: 'ip', + description: + 'Translated ip of destination based NAT sessions (e.g. internet\nto private DMZ)\n\nTypically used with load balancers, firewalls, or routers.', + }, + { + name: 'nat.port', + level: 'extended', + type: 'long', + format: 'string', + description: + 'Port the source session is translated to by NAT Device.\n\nTypically used with load balancers, firewalls, or routers.', + }, + { + name: 'packets', + level: 'core', + type: 'long', + description: 'Packets sent from the destination to the source.', + example: 12, + }, + { + name: 'port', + level: 'core', + type: 'long', + format: 'string', + description: 'Port of the destination.', + }, + { + name: 'registered_domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The highest registered destination domain, stripped of the subdomain.\n\nFor example, the registered domain for "foo.google.com" is "google.com".\n\nThis value can be determined precisely with a list like the public suffix\nlist (http://publicsuffix.org). Trying to approximate this by simply taking\nthe last two labels will not work well for TLDs such as "co.uk".', + example: 'google.com', + }, + { + name: 'top_level_domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The effective top level domain (eTLD), also known as the domain\nsuffix, is the last part of the domain name. For example, the top level domain\nfor google.com is "com".\n\nThis value can be determined precisely with a list like the public suffix\nlist (http://publicsuffix.org). Trying to approximate this by simply taking\nthe last label will not work well for effective TLDs such as "co.uk".', + example: 'co.uk', + }, + { + name: 'user.domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the directory the user is a member of.\n\nFor example, an LDAP or Active Directory domain name.', + }, + { + name: 'user.email', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'User email address.', + }, + { + name: 'user.full_name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: "User's full name, if available.", + example: 'Albert Einstein', + }, + { + name: 'user.group.domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the directory the group is a member of.\n\nFor example, an LDAP or Active Directory domain name.', + }, + { + name: 'user.group.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifier for the group on the system/platform.', + }, + { + name: 'user.group.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the group.', + }, + { + name: 'user.hash', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Unique user hash to correlate information for a user in anonymized\nform.\n\nUseful if `user.id` or `user.name` contain confidential information and cannot\nbe used.', + }, + { + name: 'user.id', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifier of the user.', + }, + { + name: 'user.name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Short name or login of the user.', + example: 'albert', + }, + ], + }, + { + name: 'dll', + title: 'DLL', group: 2, - level: 'core', - name: 'event.category', - required: false, - type: 'keyword', - }, - 'event.created': { description: - 'event.created contains the date when the event was created.\nThis timestamp is distinct from @timestamp in that @timestamp contains the processed timestamp. For logs these two timestamps can be different as the timestamp in the log line and when the event is read for example by Filebeat are not identical. `@timestamp` must contain the timestamp extracted from the log line, event.created when the log line is read. The same could apply to package capturing where @timestamp contains the timestamp extracted from the network package and event.created when the event was created.\nIn case the two timestamps are identical, @timestamp should be used.', - example: '', - footnote: '', + 'These fields contain information about code libraries dynamically\nloaded into processes.\n\n\nMany operating systems refer to "shared code libraries" with different names,\nbut this field set refers to all of the following:\n\n* Dynamic-link library (`.dll`) commonly used on Windows\n\n* Shared Object (`.so`) commonly used on Unix-like operating systems\n\n* Dynamic library (`.dylib`) commonly used on macOS', + type: 'group', + fields: [ + { + name: 'code_signature.exists', + level: 'core', + type: 'boolean', + description: 'Boolean to capture if a signature is present.', + example: 'true', + default_field: false, + }, + { + name: 'code_signature.status', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Additional information about the certificate status.\n\nThis is useful for logging cryptographic errors with the certificate validity\nor trust status. Leave unpopulated if the validity or trust of the certificate\nwas unchecked.', + example: 'ERROR_UNTRUSTED_ROOT', + default_field: false, + }, + { + name: 'code_signature.subject_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Subject name of the code signer', + example: 'Microsoft Corporation', + default_field: false, + }, + { + name: 'code_signature.trusted', + level: 'extended', + type: 'boolean', + description: + 'Stores the trust status of the certificate chain.\n\nValidating the trust of the certificate chain may be complicated, and this\nfield should only be populated by tools that actively check the status.', + example: 'true', + default_field: false, + }, + { + name: 'code_signature.valid', + level: 'extended', + type: 'boolean', + description: + 'Boolean to capture if the digital signature is verified against\nthe binary content.\n\nLeave unpopulated if a certificate was unchecked.', + example: 'true', + default_field: false, + }, + { + name: 'hash.md5', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'MD5 hash.', + default_field: false, + }, + { + name: 'hash.sha1', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'SHA1 hash.', + default_field: false, + }, + { + name: 'hash.sha256', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'SHA256 hash.', + default_field: false, + }, + { + name: 'hash.sha512', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'SHA512 hash.', + default_field: false, + }, + { + name: 'name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the library.\n\nThis generally maps to the name of the file on disk.', + example: 'kernel32.dll', + default_field: false, + }, + { + name: 'path', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Full file path of the library.', + example: 'C:\\Windows\\System32\\kernel32.dll', + default_field: false, + }, + { + name: 'pe.architecture', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'CPU architecture target for the file.', + example: 'x64', + default_field: false, + }, + { + name: 'pe.company', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal company name of the file, provided at compile-time.', + example: 'Microsoft Corporation', + default_field: false, + }, + { + name: 'pe.description', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal description of the file, provided at compile-time.', + example: 'Paint', + default_field: false, + }, + { + name: 'pe.file_version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal version of the file, provided at compile-time.', + example: '6.3.9600.17415', + default_field: false, + }, + { + name: 'pe.imphash', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'A hash of the imports in a PE file. An imphash -- or import hash\n-- can be used to fingerprint binaries even after recompilation or other code-level\ntransformations have occurred, which would change more traditional hash values.\n\nLearn more at https://www.fireeye.com/blog/threat-research/2014/01/tracking-malware-import-hashing.html.', + example: '0c6803c4e922103c4dca5963aad36ddf', + default_field: false, + }, + { + name: 'pe.original_file_name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal name of the file, provided at compile-time.', + example: 'MSPAINT.EXE', + default_field: false, + }, + { + name: 'pe.product', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal product name of the file, provided at compile-time.', + example: 'Microsoft® Windows® Operating System', + default_field: false, + }, + ], + }, + { + name: 'dns', + title: 'DNS', group: 2, - level: 'core', - name: 'event.created', - required: false, - type: 'date', - }, - 'event.dataset': { description: - 'Name of the dataset.\nThe concept of a `dataset` (fileset / metricset) is used in Beats as a subset of modules. It contains the information which is currently stored in metricset.name and metricset.module or fileset.name.', - example: 'stats', - footnote: '', + 'Fields describing DNS queries and answers.\n\nDNS events should either represent a single DNS query prior to getting answers\n(`dns.type:query`) or they should represent a full exchange and contain the\nquery details as well as all of the answers that were provided for this query\n(`dns.type:answer`).', + type: 'group', + fields: [ + { + name: 'answers', + level: 'extended', + type: 'object', + object_type: 'keyword', + description: + 'An array containing an object for each answer section returned\nby the server.\n\nThe main keys that should be present in these objects are defined by ECS.\nRecords that have more information may contain more keys than what ECS defines.\n\nNot all DNS data sources give all details about DNS answers. At minimum, answer\nobjects must contain the `data` key. If more information is available, map\nas much of it to ECS as possible, and add any additional fields to the answer\nobjects as custom fields.', + }, + { + name: 'answers.class', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'The class of DNS data contained in this resource record.', + example: 'IN', + }, + { + name: 'answers.data', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The data describing the resource.\n\nThe meaning of this data depends on the type and class of the resource record.', + example: '10.10.10.10', + }, + { + name: 'answers.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The domain name to which this resource record pertains.\n\nIf a chain of CNAME is being resolved, each answer `name` should be the\none that corresponds with the answer `data`. It should not simply be the\noriginal `question.name` repeated.', + example: 'www.google.com', + }, + { + name: 'answers.ttl', + level: 'extended', + type: 'long', + description: + 'The time interval in seconds that this resource record may be cached\nbefore it should be discarded. Zero values mean that the data should not be\ncached.', + example: 180, + }, + { + name: 'answers.type', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'The type of data contained in this resource record.', + example: 'CNAME', + }, + { + name: 'header_flags', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Array of 2 letter DNS header flags.\n\nExpected values are: AA, TC, RD, RA, AD, CD, DO.', + example: ['RD', 'RA'], + }, + { + name: 'id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The DNS packet identifier assigned by the program that generated\nthe query. The identifier is copied to the response.', + example: 62111, + }, + { + name: 'op_code', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The DNS operation code that specifies the kind of query in the\nmessage. This value is set by the originator of a query and copied into the\nresponse.', + example: 'QUERY', + }, + { + name: 'question.class', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'The class of records being queried.', + example: 'IN', + }, + { + name: 'question.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The name being queried.\n\nIf the name field contains non-printable characters (below 32 or above 126),\nthose characters should be represented as escaped base 10 integers (\\DDD).\nBack slashes and quotes should be escaped. Tabs, carriage returns, and line\nfeeds should be converted to \\t, \\r, and \\n respectively.', + example: 'www.google.com', + }, + { + name: 'question.registered_domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The highest registered domain, stripped of the subdomain.\n\nFor example, the registered domain for "foo.google.com" is "google.com".\n\nThis value can be determined precisely with a list like the public suffix\nlist (http://publicsuffix.org). Trying to approximate this by simply taking\nthe last two labels will not work well for TLDs such as "co.uk".', + example: 'google.com', + }, + { + name: 'question.subdomain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The subdomain is all of the labels under the registered_domain.\n\nIf the domain has multiple levels of subdomain, such as "sub2.sub1.example.com",\nthe subdomain field should contain "sub2.sub1", with no trailing period.', + example: 'www', + }, + { + name: 'question.top_level_domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The effective top level domain (eTLD), also known as the domain\nsuffix, is the last part of the domain name. For example, the top level domain\nfor google.com is "com".\n\nThis value can be determined precisely with a list like the public suffix\nlist (http://publicsuffix.org). Trying to approximate this by simply taking\nthe last label will not work well for effective TLDs such as "co.uk".', + example: 'co.uk', + }, + { + name: 'question.type', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'The type of record being queried.', + example: 'AAAA', + }, + { + name: 'resolved_ip', + level: 'extended', + type: 'ip', + description: + 'Array containing all IPs seen in `answers.data`.\n\nThe `answers` array can be difficult to use, because of the variety of data\nformats it can contain. Extracting all IP addresses seen in there to `dns.resolved_ip`\nmakes it possible to index them as IP addresses, and makes them easier to\nvisualize and query for.', + example: ['10.10.10.10', '10.10.10.11'], + }, + { + name: 'response_code', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'The DNS response code.', + example: 'NOERROR', + }, + { + name: 'type', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The type of DNS event captured, query or answer.\n\nIf your source of DNS events only gives you DNS queries, you should only create\ndns events of type `dns.type:query`.\n\nIf your source of DNS events gives you answers as well, you should create\none event per query (optionally as soon as the query is seen). And a second\nevent containing all query details as well as an array of answers.', + example: 'answer', + }, + ], + }, + { + name: 'ecs', + title: 'ECS', + group: 2, + description: 'Meta-information specific to ECS.', + type: 'group', + fields: [ + { + name: 'version', + level: 'core', + required: true, + type: 'keyword', + ignore_above: 1024, + description: + 'ECS version this event conforms to. `ecs.version` is a required\nfield and must exist in all events.\n\nWhen querying across multiple indices -- which may conform to slightly different\nECS versions -- this field lets integrations adjust to the schema version\nof the events.', + example: '1.0.0', + }, + ], + }, + { + name: 'error', + title: 'Error', group: 2, - level: 'core', - name: 'event.dataset', - required: false, - type: 'keyword', - }, - 'event.duration': { description: - 'Duration of the event in nanoseconds.\nIf event.start and event.end are known this value should be the difference between the end and start time.', - example: '', - footnote: '', + 'These fields can represent errors of any kind.\n\nUse them for errors that happen while fetching events or in cases where the\nevent itself contains an error.', + type: 'group', + fields: [ + { + name: 'code', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Error code describing the error.', + }, + { + name: 'id', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifier for the error.', + }, + { + name: 'message', + level: 'core', + type: 'text', + description: 'Error message.', + }, + { + name: 'stack_trace', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'The stack trace of this error in plain text.', + }, + { + name: 'type', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'The type of the error, for example the class name of the exception.', + example: 'java.lang.NullPointerException', + }, + ], + }, + { + name: 'event', + title: 'Event', group: 2, - level: 'core', - name: 'event.duration', - required: false, - type: 'long', - }, - 'event.end': { description: - 'event.end contains the date when the event ended or when the activity was last observed.', - example: '', - footnote: '', + 'The event fields are used for context information about the log\nor metric event itself.\n\nA log is defined as an event containing details of something that happened.\nLog events must include the time at which the thing happened. Examples of log\nevents include a process starting on a host, a network packet being sent from\na source to a destination, or a network connection between a client and a server\nbeing initiated or closed. A metric is defined as an event containing one or\nmore numerical measurements and the time at which the measurement was taken.\nExamples of metric events include memory pressure measured on a host and device\ntemperature. See the `event.kind` definition in this section for additional\ndetails about metric and state events.', + type: 'group', + fields: [ + { + name: 'action', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'The action captured by the event.\n\nThis describes the information in the event. It is more specific than `event.category`.\nExamples are `group-add`, `process-started`, `file-created`. The value is\nnormally defined by the implementer.', + example: 'user-password-change', + }, + { + name: 'category', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'This is one of four ECS Categorization Fields, and indicates the\nsecond level in the ECS category hierarchy.\n\n`event.category` represents the "big buckets" of ECS categories. For example,\nfiltering on `event.category:process` yields all events relating to process\nactivity. This field is closely related to `event.type`, which is used as\na subcategory.\n\nThis field is an array. This will allow proper categorization of some events\nthat fall in multiple categories.', + example: 'authentication', + }, + { + name: 'code', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Identification code for this event, if one exists.\n\nSome event sources use event codes to identify messages unambiguously, regardless\nof message language or wording adjustments over time. An example of this is\nthe Windows Event ID.', + example: 4648, + }, + { + name: 'created', + level: 'core', + type: 'date', + description: + 'event.created contains the date/time when the event was first\nread by an agent, or by your pipeline.\n\nThis field is distinct from @timestamp in that @timestamp typically contain\nthe time extracted from the original event.\n\nIn most situations, these two timestamps will be slightly different. The difference\ncan be used to calculate the delay between your source generating an event,\nand the time when your agent first processed it. This can be used to monitor\nyour agent or pipeline ability to keep up with your event source.\n\nIn case the two timestamps are identical, @timestamp should be used.', + example: '2016-05-23T08:05:34.857Z', + }, + { + name: 'dataset', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the dataset.\n\nIf an event source publishes more than one type of log or events (e.g. access\nlog, error log), the dataset is used to specify which one the event comes\nfrom.\n\nIt is recommended but not required to start the dataset name with the module\nname, followed by a dot, then the dataset name.', + example: 'apache.access', + }, + { + name: 'duration', + level: 'core', + type: 'long', + format: 'duration', + input_format: 'nanoseconds', + output_format: 'asMilliseconds', + output_precision: 1, + description: + 'Duration of the event in nanoseconds.\n\nIf event.start and event.end are known this value should be the difference\nbetween the end and start time.', + }, + { + name: 'end', + level: 'extended', + type: 'date', + description: + 'event.end contains the date when the event ended or when the activity\nwas last observed.', + }, + { + name: 'hash', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Hash (perhaps logstash fingerprint) of raw field to be able to\ndemonstrate log integrity.', + example: '123456789012345678901234567890ABCD', + }, + { + name: 'id', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Unique ID to describe the event.', + example: '8a4f500d', + }, + { + name: 'ingested', + level: 'core', + type: 'date', + description: + 'Timestamp when an event arrived in the central data store.\n\nThis is different from `@timestamp`, which is when the event originally occurred. It is\nalso different from `event.created`, which is meant to capture the first time\nan agent saw the event.\n\nIn normal conditions, assuming no tampering, the timestamps should chronologically\nlook like this: `@timestamp` < `event.created` < `event.ingested`.', + example: '2016-05-23T08:05:35.101Z', + default_field: false, + }, + { + name: 'kind', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'This is one of four ECS Categorization Fields, and indicates the\nhighest level in the ECS category hierarchy.\n\n`event.kind` gives high-level information about what type of information the\nevent contains, without being specific to the contents of the event. For example,\nvalues of this field distinguish alert events from metric events.\n\nThe value of this field can be used to inform how these kinds of events should\nbe handled. They may warrant different retention, different access control,\nit may also help understand whether the data coming in at a regular interval\nor not.', + example: 'alert', + }, + { + name: 'module', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the module this data is coming from.\n\nIf your monitoring agent supports the concept of modules or plugins to process\nevents of a given source (e.g. Apache logs), `event.module` should contain\nthe name of this module.', + example: 'apache', + }, + { + name: 'original', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'Raw text message of entire event. Used to demonstrate log integrity.\n\nThis field is not indexed and doc_values are disabled. It cannot be searched,\nbut it can be retrieved from `_source`.', + example: + 'Sep 19 08:26:10 host CEF:0|Security| threatmanager|1.0|100|\nworm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2spt=1232', + }, + { + name: 'outcome', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'This is one of four ECS Categorization Fields, and indicates the\nlowest level in the ECS category hierarchy.\n\n`event.outcome` simply denotes whether the event represents a success or a\nfailure from the perspective of the entity that produced the event.\n\nNote that when a single transaction is described in multiple events, each\nevent may populate different values of `event.outcome`, according to their\nperspective.\n\nAlso note that in the case of a compound event (a single event that contains\nmultiple logical events), this field should be populated with the value that\nbest captures the overall success or failure from the perspective of the event\nproducer.\n\nFurther note that not all events will have an associated outcome. For example,\nthis field is generally not populated for metric events, events with `event.type:info`,\nor any events for which an outcome does not make logical sense.', + example: 'success', + }, + { + name: 'provider', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Source of the event.\n\nEvent transports such as Syslog or the Windows Event Log typically mention\nthe source of an event. It can be the name of the software that generated\nthe event (e.g. Sysmon, httpd), or of a subsystem of the operating system\n(kernel, Microsoft-Windows-Security-Auditing).', + example: 'kernel', + }, + { + name: 'reference', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Reference URL linking to additional information about this event.\n\nThis URL links to a static definition of the this event. Alert events, indicated\nby `event.kind:alert`, are a common use case for this field.', + example: 'https://system.vendor.com/event/#0001234', + default_field: false, + }, + { + name: 'risk_score', + level: 'core', + type: 'float', + description: + "Risk score or priority of the event (e.g. security solutions).\nUse your system's original value here.", + }, + { + name: 'risk_score_norm', + level: 'extended', + type: 'float', + description: + 'Normalized risk score or priority of the event, on a scale of\n0 to 100.\n\nThis is mainly useful if you use more than one system that assigns risk scores,\nand you want to see a normalized value across all systems.', + }, + { + name: 'sequence', + level: 'extended', + type: 'long', + format: 'string', + description: + 'Sequence number of the event.\n\nThe sequence number is a value published by some event sources, to make the\nexact ordering of events unambiguous, regardless of the timestamp precision.', + }, + { + name: 'severity', + level: 'core', + type: 'long', + format: 'string', + description: + 'The numeric severity of the event according to your event source.\n\nWhat the different severity values mean can be different between sources and\nuse cases. It is up to the implementer to make sure severities are consistent\nacross events from the same source.\n\nThe Syslog severity belongs in `log.syslog.severity.code`. `event.severity`\nis meant to represent the severity according to the event source (e.g. firewall,\nIDS). If the event source does not publish its own severity, you may optionally\ncopy the `log.syslog.severity.code` to `event.severity`.', + example: 7, + }, + { + name: 'start', + level: 'extended', + type: 'date', + description: + 'event.start contains the date when the event started or when the\nactivity was first observed.', + }, + { + name: 'timezone', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'This field should be populated when the event timestamp does\nnot include timezone information already (e.g. default Syslog timestamps).\nIt is optional otherwise.\n\nAcceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"),\nabbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00").', + }, + { + name: 'type', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'This is one of four ECS Categorization Fields, and indicates the\nthird level in the ECS category hierarchy.\n\n`event.type` represents a categorization "sub-bucket" that, when used along\nwith the `event.category` field values, enables filtering events down to a\nlevel appropriate for single visualization.\n\nThis field is an array. This will allow proper categorization of some events\nthat fall in multiple event types.', + }, + { + name: 'url', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'URL linking to an external system to continue investigation of\nthis event.\n\nThis URL links to another system where in-depth investigation of the specific\noccurence of this event can take place. Alert events, indicated by `event.kind:alert`,\nare a common use case for this field.', + example: 'https://mysystem.mydomain.com/alert/5271dedb-f5b0-4218-87f0-4ac4870a38fe', + default_field: false, + }, + ], + }, + { + name: 'file', + title: 'File', group: 2, - level: 'extended', - name: 'event.end', - required: false, - type: 'date', - }, - 'event.hash': { description: - 'Hash (perhaps logstash fingerprint) of raw field to be able to demonstrate log integrity.', - example: '123456789012345678901234567890ABCD', - footnote: '', - group: 2, - level: 'extended', - name: 'event.hash', - required: false, - type: 'keyword', - }, - 'event.id': { - description: 'Unique ID to describe the event.', - example: '8a4f500d', - footnote: '', + 'A file is defined as a set of information that has been created\non, or has existed on a filesystem.\n\nFile objects can be associated with host events, network events, and/or file\nevents (e.g., those produced by File Integrity Monitoring [FIM] products or\nservices). File fields provide details about the affected file associated with\nthe event or metric.', + type: 'group', + fields: [ + { + name: 'accessed', + level: 'extended', + type: 'date', + description: + 'Last time the file was accessed.\n\nNote that not all filesystems keep track of access time.', + }, + { + name: 'attributes', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Array of file attributes.\n\nAttributes names will vary by platform. Here is a non-exhaustive list of values\nthat are expected in this field: archive, compressed, directory, encrypted,\nexecute, hidden, read, readonly, system, write.', + example: '["readonly", "system"]', + default_field: false, + }, + { + name: 'code_signature.exists', + level: 'core', + type: 'boolean', + description: 'Boolean to capture if a signature is present.', + example: 'true', + default_field: false, + }, + { + name: 'code_signature.status', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Additional information about the certificate status.\n\nThis is useful for logging cryptographic errors with the certificate validity\nor trust status. Leave unpopulated if the validity or trust of the certificate\nwas unchecked.', + example: 'ERROR_UNTRUSTED_ROOT', + default_field: false, + }, + { + name: 'code_signature.subject_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Subject name of the code signer', + example: 'Microsoft Corporation', + default_field: false, + }, + { + name: 'code_signature.trusted', + level: 'extended', + type: 'boolean', + description: + 'Stores the trust status of the certificate chain.\n\nValidating the trust of the certificate chain may be complicated, and this\nfield should only be populated by tools that actively check the status.', + example: 'true', + default_field: false, + }, + { + name: 'code_signature.valid', + level: 'extended', + type: 'boolean', + description: + 'Boolean to capture if the digital signature is verified against\nthe binary content.\n\nLeave unpopulated if a certificate was unchecked.', + example: 'true', + default_field: false, + }, + { + name: 'created', + level: 'extended', + type: 'date', + description: + 'File creation time.\n\nNote that not all filesystems store the creation time.', + }, + { + name: 'ctime', + level: 'extended', + type: 'date', + description: + 'Last time the file attributes or metadata changed.\n\nNote that changes to the file content will update `mtime`. This implies `ctime`\nwill be adjusted at the same time, since `mtime` is an attribute of the file.', + }, + { + name: 'device', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Device that is the source of the file.', + example: 'sda', + }, + { + name: 'directory', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Directory where the file is located. It should include the drive\nletter, when appropriate.', + example: '/home/alice', + }, + { + name: 'drive_letter', + level: 'extended', + type: 'keyword', + ignore_above: 1, + description: + 'Drive letter where the file is located. This field is only relevant\non Windows.\n\nThe value should be uppercase, and not include the colon.', + example: 'C', + default_field: false, + }, + { + name: 'extension', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'File extension.', + example: 'png', + }, + { + name: 'gid', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Primary group ID (GID) of the file.', + example: '1001', + }, + { + name: 'group', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Primary group name of the file.', + example: 'alice', + }, + { + name: 'hash.md5', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'MD5 hash.', + }, + { + name: 'hash.sha1', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'SHA1 hash.', + }, + { + name: 'hash.sha256', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'SHA256 hash.', + }, + { + name: 'hash.sha512', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'SHA512 hash.', + }, + { + name: 'inode', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Inode representing the file in the filesystem.', + example: '256383', + }, + { + name: 'mime_type', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'MIME type should identify the format of the file or stream of bytes\nusing https://www.iana.org/assignments/media-types/media-types.xhtml[IANA\nofficial types], where possible. When more than one type is applicable, the\nmost specific type should be used.', + default_field: false, + }, + { + name: 'mode', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Mode of the file in octal representation.', + example: '0640', + }, + { + name: 'mtime', + level: 'extended', + type: 'date', + description: 'Last time the file content was modified.', + }, + { + name: 'name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the file including the extension, without the directory.', + example: 'example.png', + }, + { + name: 'owner', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: "File owner's username.", + example: 'alice', + }, + { + name: 'path', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: + 'Full path to the file, including the file name. It should include\nthe drive letter, when appropriate.', + example: '/home/alice/example.png', + }, + { + name: 'pe.architecture', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'CPU architecture target for the file.', + example: 'x64', + default_field: false, + }, + { + name: 'pe.company', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal company name of the file, provided at compile-time.', + example: 'Microsoft Corporation', + default_field: false, + }, + { + name: 'pe.description', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal description of the file, provided at compile-time.', + example: 'Paint', + default_field: false, + }, + { + name: 'pe.file_version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal version of the file, provided at compile-time.', + example: '6.3.9600.17415', + default_field: false, + }, + { + name: 'pe.imphash', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'A hash of the imports in a PE file. An imphash -- or import hash\n-- can be used to fingerprint binaries even after recompilation or other code-level\ntransformations have occurred, which would change more traditional hash values.\n\nLearn more at https://www.fireeye.com/blog/threat-research/2014/01/tracking-malware-import-hashing.html.', + example: '0c6803c4e922103c4dca5963aad36ddf', + default_field: false, + }, + { + name: 'pe.original_file_name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal name of the file, provided at compile-time.', + example: 'MSPAINT.EXE', + default_field: false, + }, + { + name: 'pe.product', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal product name of the file, provided at compile-time.', + example: 'Microsoft® Windows® Operating System', + default_field: false, + }, + { + name: 'size', + level: 'extended', + type: 'long', + description: 'File size in bytes.\n\nOnly relevant when `file.type` is "file".', + example: 16384, + }, + { + name: 'target_path', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Target path for symlinks.', + }, + { + name: 'type', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'File type (file, dir, or symlink).', + example: 'file', + }, + { + name: 'uid', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'The user ID (UID) or security identifier (SID) of the file owner.', + example: '1001', + }, + ], + }, + { + name: 'geo', + title: 'Geo', group: 2, - level: 'core', - name: 'event.id', - required: false, - type: 'keyword', - }, - 'event.kind': { description: - 'The kind of the event.\nThis gives information about what type of information the event contains, without being specific to the contents of the event. Examples are `event`, `state`, `alarm`. Warning: In future versions of ECS, we plan to provide a list of acceptable values for this field, please use with caution.', - example: 'state', - footnote: '', + 'Geo fields can carry data about a specific location related to an\nevent.\n\nThis geolocation information can be derived from techniques such as Geo IP,\nor be user-supplied.', + type: 'group', + fields: [ + { + name: 'city_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'City name.', + example: 'Montreal', + }, + { + name: 'continent_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the continent.', + example: 'North America', + }, + { + name: 'country_iso_code', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Country ISO code.', + example: 'CA', + }, + { + name: 'country_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Country name.', + example: 'Canada', + }, + { + name: 'location', + level: 'core', + type: 'geo_point', + description: 'Longitude and latitude.', + example: '{ "lon": -73.614830, "lat": 45.505918 }', + }, + { + name: 'name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'User-defined description of a location, at the level of granularity\nthey care about.\n\nCould be the name of their data centers, the floor number, if this describes\na local physical entity, city names.\n\nNot typically used in automated geolocation.', + example: 'boston-dc', + }, + { + name: 'region_iso_code', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Region ISO code.', + example: 'CA-QC', + }, + { + name: 'region_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Region name.', + example: 'Quebec', + }, + ], + }, + { + name: 'group', + title: 'Group', group: 2, - level: 'extended', - name: 'event.kind', - required: false, - type: 'keyword', - }, - 'event.module': { description: - 'Name of the module this data is coming from.\nThis information is coming from the modules used in Beats or Logstash.', - example: 'mysql', - footnote: '', + 'The group fields are meant to represent groups that are relevant\nto the event.', + type: 'group', + fields: [ + { + name: 'domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the directory the group is a member of.\n\nFor example, an LDAP or Active Directory domain name.', + }, + { + name: 'id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifier for the group on the system/platform.', + }, + { + name: 'name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the group.', + }, + ], + }, + { + name: 'hash', + title: 'Hash', group: 2, - level: 'core', - name: 'event.module', - required: false, - type: 'keyword', - }, - 'event.original': { description: - 'Raw text message of entire event. Used to demonstrate log integrity.\nThis field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`.', - example: - 'Sep 19 08:26:10 host CEF:0|Security| threatmanager|1.0|100| worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2spt=1232', - footnote: '', + 'The hash fields represent different hash algorithms and their values.\n\nField names for common hashes (e.g. MD5, SHA1) are predefined. Add fields for\nother hashes by lowercasing the hash algorithm name and using underscore separators\nas appropriate (snake case, e.g. sha3_512).', + type: 'group', + fields: [ + { + name: 'md5', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'MD5 hash.', + }, + { + name: 'sha1', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'SHA1 hash.', + }, + { + name: 'sha256', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'SHA256 hash.', + }, + { + name: 'sha512', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'SHA512 hash.', + }, + ], + }, + { + name: 'host', + title: 'Host', group: 2, - level: 'core', - name: 'event.original', - required: false, - type: '(not indexed)', - }, - 'event.outcome': { description: - 'The outcome of the event.\nIf the event describes an action, this fields contains the outcome of that action. Examples outcomes are `success` and `failure`. Warning: In future versions of ECS, we plan to provide a list of acceptable values for this field, please use with caution.', - example: 'success', - footnote: '', + 'A host is defined as a general computing instance.\n\nECS host.* fields should be populated with details about the host on which the\nevent happened, or from which the measurement was taken. Host types include\nhardware, virtual machines, Docker containers, and Kubernetes nodes.', + type: 'group', + fields: [ + { + name: 'architecture', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Operating system architecture.', + example: 'x86_64', + }, + { + name: 'domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the domain of which the host is a member.\n\nFor example, on Windows this could be the host Active Directory domain\nor NetBIOS domain name. For Linux this could be the domain of the host\nLDAP provider.', + example: 'CONTOSO', + default_field: false, + }, + { + name: 'geo.city_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'City name.', + example: 'Montreal', + }, + { + name: 'geo.continent_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the continent.', + example: 'North America', + }, + { + name: 'geo.country_iso_code', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Country ISO code.', + example: 'CA', + }, + { + name: 'geo.country_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Country name.', + example: 'Canada', + }, + { + name: 'geo.location', + level: 'core', + type: 'geo_point', + description: 'Longitude and latitude.', + example: '{ "lon": -73.614830, "lat": 45.505918 }', + }, + { + name: 'geo.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'User-defined description of a location, at the level of granularity\nthey care about.\n\nCould be the name of their data centers, the floor number, if this describes\na local physical entity, city names.\n\nNot typically used in automated geolocation.', + example: 'boston-dc', + }, + { + name: 'geo.region_iso_code', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Region ISO code.', + example: 'CA-QC', + }, + { + name: 'geo.region_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Region name.', + example: 'Quebec', + }, + { + name: 'hostname', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'Hostname of the host.\n\nIt normally contains what the `hostname` command returns on the host machine.', + }, + { + name: 'id', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'Unique host id.\n\nAs hostname is not always unique, use values that are meaningful in your environment.\n\nExample: The current usage of `beat.name`.', + }, + { + name: 'ip', + level: 'core', + type: 'ip', + description: 'Host ip addresses.', + }, + { + name: 'mac', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Host mac addresses.', + }, + { + name: 'name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the host.\n\nIt can contain what `hostname` returns on Unix systems, the fully qualified\ndomain name, or a name specified by the user. The sender decides which value\nto use.', + }, + { + name: 'os.family', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'OS family (such as redhat, debian, freebsd, windows).', + example: 'debian', + }, + { + name: 'os.full', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Operating system name, including the version or code name.', + example: 'Mac OS Mojave', + }, + { + name: 'os.kernel', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Operating system kernel version as a raw string.', + example: '4.4.0-112-generic', + }, + { + name: 'os.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Operating system name, without the version.', + example: 'Mac OS X', + }, + { + name: 'os.platform', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Operating system platform (such centos, ubuntu, windows).', + example: 'darwin', + }, + { + name: 'os.version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Operating system version as a raw string.', + example: '10.14.1', + }, + { + name: 'type', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'Type of host.\n\nFor Cloud providers this can be the machine type like `t2.medium`. If vm,\nthis could be the container, for example, or other information meaningful\nin your environment.', + }, + { + name: 'uptime', + level: 'extended', + type: 'long', + description: 'Seconds the host has been up.', + example: 1325, + }, + { + name: 'user.domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the directory the user is a member of.\n\nFor example, an LDAP or Active Directory domain name.', + }, + { + name: 'user.email', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'User email address.', + }, + { + name: 'user.full_name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: "User's full name, if available.", + example: 'Albert Einstein', + }, + { + name: 'user.group.domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the directory the group is a member of.\n\nFor example, an LDAP or Active Directory domain name.', + }, + { + name: 'user.group.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifier for the group on the system/platform.', + }, + { + name: 'user.group.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the group.', + }, + { + name: 'user.hash', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Unique user hash to correlate information for a user in anonymized\nform.\n\nUseful if `user.id` or `user.name` contain confidential information and cannot\nbe used.', + }, + { + name: 'user.id', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifier of the user.', + }, + { + name: 'user.name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Short name or login of the user.', + example: 'albert', + }, + ], + }, + { + name: 'http', + title: 'HTTP', group: 2, - level: 'extended', - name: 'event.outcome', - required: false, - type: 'keyword', - }, - 'event.risk_score': { description: - "Risk score or priority of the event (e.g. security solutions). Use your system's original value here.", - example: '', - footnote: '', + 'Fields related to HTTP activity. Use the `url` field set to store\nthe url of the request.', + type: 'group', + fields: [ + { + name: 'request.body.bytes', + level: 'extended', + type: 'long', + format: 'bytes', + description: 'Size in bytes of the request body.', + example: 887, + }, + { + name: 'request.body.content', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'The full HTTP request body.', + example: 'Hello world', + }, + { + name: 'request.bytes', + level: 'extended', + type: 'long', + format: 'bytes', + description: 'Total size in bytes of the request (body and headers).', + example: 1437, + }, + { + name: 'request.method', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'HTTP request method.\n\nThe field value must be normalized to lowercase for querying. See the documentation\nsection "Implementing ECS".', + example: 'get, post, put', + }, + { + name: 'request.referrer', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Referrer for this HTTP request.', + example: 'https://blog.example.com/', + }, + { + name: 'response.body.bytes', + level: 'extended', + type: 'long', + format: 'bytes', + description: 'Size in bytes of the response body.', + example: 887, + }, + { + name: 'response.body.content', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'The full HTTP response body.', + example: 'Hello world', + }, + { + name: 'response.bytes', + level: 'extended', + type: 'long', + format: 'bytes', + description: 'Total size in bytes of the response (body and headers).', + example: 1437, + }, + { + name: 'response.status_code', + level: 'extended', + type: 'long', + format: 'string', + description: 'HTTP response status code.', + example: 404, + }, + { + name: 'version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'HTTP version.', + example: 1.1, + }, + ], + }, + { + name: 'interface', + title: 'Interface', group: 2, - level: 'core', - name: 'event.risk_score', - required: false, - type: 'float', - }, - 'event.risk_score_norm': { description: - 'Normalized risk score or priority of the event, on a scale of 0 to 100.\nThis is mainly useful if you use more than one system that assigns risk scores, and you want to see a normalized value across all systems.', - example: '', - footnote: '', + 'The interface fields are used to record ingress and egress interface\ninformation when reported by an observer (e.g. firewall, router, load balancer)\nin the context of the observer handling a network connection. In the case of\na single observer interface (e.g. network sensor on a span port) only the observer.ingress\ninformation should be populated.', + type: 'group', + fields: [ + { + name: 'alias', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Interface alias as reported by the system, typically used in firewall\nimplementations for e.g. inside, outside, or dmz logical interface naming.', + example: 'outside', + default_field: false, + }, + { + name: 'id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Interface ID as reported by an observer (typically SNMP interface\nID).', + example: 10, + default_field: false, + }, + { + name: 'name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Interface name as reported by the system.', + example: 'eth0', + default_field: false, + }, + ], + }, + { + name: 'log', + title: 'Log', group: 2, - level: 'extended', - name: 'event.risk_score_norm', - required: false, - type: 'float', - }, - 'event.severity': { description: - "Severity describes the severity of the event. What the different severity values mean can very different between use cases. It's up to the implementer to make sure severities are consistent across events.", - example: '7', - footnote: '', + 'Details about the event logging mechanism or logging transport.\n\nThe log.* fields are typically populated with details about the logging mechanism\nused to create and/or transport the event. For example, syslog details belong\nunder `log.syslog.*`.\n\nThe details specific to your event source are typically not logged under `log.*`,\nbut rather in `event.*` or in other ECS fields.', + type: 'group', + fields: [ + { + name: 'file.path', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + "Full path to the log file this event came from, including the\nfile name. It should include the drive letter, when appropriate.\n\nIf the event wasn't read from a log file, do not populate this field.", + example: '/var/log/fun-times.log', + default_field: false, + }, + { + name: 'level', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'Original log level of the log event.\n\nIf the source of the event provides a log level or textual severity, this\nis the one that goes in `log.level`. If your source does not specify one,\nyou may put your event transport severity here (e.g. Syslog severity).\n\nSome examples are `warn`, `err`, `i`, `informational`.', + example: 'error', + }, + { + name: 'logger', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'The name of the logger inside an application. This is usually the\nname of the class which initialized the logger, or can be a custom name.', + example: 'org.elasticsearch.bootstrap.Bootstrap', + }, + { + name: 'origin.file.line', + level: 'extended', + type: 'integer', + description: + 'The line number of the file containing the source code which originated\nthe log event.', + example: 42, + }, + { + name: 'origin.file.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The name of the file containing the source code which originated\nthe log event.\n\nNote that this field is not meant to capture the log file. The correct field\nto capture the log file is `log.file.path`.', + example: 'Bootstrap.java', + }, + { + name: 'origin.function', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'The name of the function or method which originated the log event.', + example: 'init', + }, + { + name: 'original', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'This is the original log message and contains the full log message\nbefore splitting it up in multiple parts.\n\nIn contrast to the `message` field which can contain an extracted part of\nthe log message, this field contains the original, full log message. It can\nhave already some modifications applied like encoding or new lines removed\nto clean up the log message.\n\nThis field is not indexed and doc_values are disabled so it cannot be queried\nbut the value can be retrieved from `_source`.', + example: 'Sep 19 08:26:10 localhost My log', + }, + { + name: 'syslog', + level: 'extended', + type: 'object', + object_type: 'keyword', + description: + 'The Syslog metadata of the event, if the event was transmitted\nvia Syslog. Please see RFCs 5424 or 3164.', + }, + { + name: 'syslog.facility.code', + level: 'extended', + type: 'long', + format: 'string', + description: + 'The Syslog numeric facility of the log event, if available.\n\nAccording to RFCs 5424 and 3164, this value should be an integer between 0\nand 23.', + example: 23, + }, + { + name: 'syslog.facility.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'The Syslog text-based facility of the log event, if available.', + example: 'local7', + }, + { + name: 'syslog.priority', + level: 'extended', + type: 'long', + format: 'string', + description: + 'Syslog numeric priority of the event, if available.\n\nAccording to RFCs 5424 and 3164, the priority is 8 * facility + severity.\nThis number is therefore expected to contain a value between 0 and 191.', + example: 135, + }, + { + name: 'syslog.severity.code', + level: 'extended', + type: 'long', + description: + 'The Syslog numeric severity of the log event, if available.\n\nIf the event source publishing via Syslog provides a different numeric severity\nvalue (e.g. firewall, IDS), your source numeric severity should go to `event.severity`.\nIf the event source does not specify a distinct severity, you can optionally\ncopy the Syslog severity to `event.severity`.', + example: 3, + }, + { + name: 'syslog.severity.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The Syslog numeric severity of the log event, if available.\n\nIf the event source publishing via Syslog provides a different severity value\n(e.g. firewall, IDS), your source text severity should go to `log.level`.\nIf the event source does not specify a distinct severity, you can optionally\ncopy the Syslog severity to `log.level`.', + example: 'Error', + }, + ], + }, + { + name: 'network', + title: 'Network', group: 2, - level: 'core', - name: 'event.severity', - required: false, - type: 'long', - }, - 'event.start': { description: - 'event.start contains the date when the event started or when the activity was first observed.', - example: '', - footnote: '', + 'The network is defined as the communication path over which a host\nor network event happens.\n\nThe network.* fields should be populated with details about the network activity\nassociated with an event.', + type: 'group', + fields: [ + { + name: 'application', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'A name given to an application level protocol. This can be arbitrarily\nassigned for things like microservices, but also apply to things like skype,\nicq, facebook, twitter. This would be used in situations where the vendor\nor service can be decoded such as from the source/dest IP owners, ports, or\nwire format.\n\nThe field value must be normalized to lowercase for querying. See the documentation\nsection "Implementing ECS".', + example: 'aim', + }, + { + name: 'bytes', + level: 'core', + type: 'long', + format: 'bytes', + description: + 'Total bytes transferred in both directions.\n\nIf `source.bytes` and `destination.bytes` are known, `network.bytes` is their\nsum.', + example: 368, + }, + { + name: 'community_id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'A hash of source and destination IPs and ports, as well as the\nprotocol used in a communication. This is a tool-agnostic standard to identify\nflows.\n\nLearn more at https://github.com/corelight/community-id-spec.', + example: '1:hO+sN4H+MG5MY/8hIrXPqc4ZQz0=', + }, + { + name: 'direction', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + "Direction of the network traffic.\nRecommended values are:\n * inbound\n * outbound\n * internal\n * external\n * unknown\n\nWhen mapping events from a host-based monitoring context, populate this field from the host's point of view.\nWhen mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of your network perimeter.", + example: 'inbound', + }, + { + name: 'forwarded_ip', + level: 'core', + type: 'ip', + description: 'Host IP address when the source IP address is the proxy.', + example: '192.1.1.2', + }, + { + name: 'iana_number', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'IANA Protocol Number (https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml).\nStandardized list of protocols. This aligns well with NetFlow and sFlow related\nlogs which use the IANA Protocol Number.', + example: 6, + }, + { + name: 'inner', + level: 'extended', + type: 'object', + object_type: 'keyword', + description: + 'Network.inner fields are added in addition to network.vlan fields\nto describe the innermost VLAN when q-in-q VLAN tagging is present. Allowed\nfields include vlan.id and vlan.name. Inner vlan fields are typically used\nwhen sending traffic with multiple 802.1q encapsulations to a network sensor\n(e.g. Zeek, Wireshark.)', + default_field: false, + }, + { + name: 'inner.vlan.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'VLAN ID as reported by the observer.', + example: 10, + default_field: false, + }, + { + name: 'inner.vlan.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Optional VLAN name as reported by the observer.', + example: 'outside', + default_field: false, + }, + { + name: 'name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Name given by operators to sections of their network.', + example: 'Guest Wifi', + }, + { + name: 'packets', + level: 'core', + type: 'long', + description: + 'Total packets transferred in both directions.\n\nIf `source.packets` and `destination.packets` are known, `network.packets`\nis their sum.', + example: 24, + }, + { + name: 'protocol', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'L7 Network protocol name. ex. http, lumberjack, transport protocol.\n\nThe field value must be normalized to lowercase for querying. See the documentation\nsection "Implementing ECS".', + example: 'http', + }, + { + name: 'transport', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'Same as network.iana_number, but instead using the Keyword name\nof the transport layer (udp, tcp, ipv6-icmp, etc.)\n\nThe field value must be normalized to lowercase for querying. See the documentation\nsection "Implementing ECS".', + example: 'tcp', + }, + { + name: 'type', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'In the OSI Model this would be the Network Layer. ipv4, ipv6,\nipsec, pim, etc\n\nThe field value must be normalized to lowercase for querying. See the documentation\nsection "Implementing ECS".', + example: 'ipv4', + }, + { + name: 'vlan.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'VLAN ID as reported by the observer.', + example: 10, + default_field: false, + }, + { + name: 'vlan.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Optional VLAN name as reported by the observer.', + example: 'outside', + default_field: false, + }, + ], + }, + { + name: 'observer', + title: 'Observer', group: 2, - level: 'extended', - name: 'event.start', - required: false, - type: 'date', - }, - 'event.timezone': { description: - 'This field should be populated when the event\'s timestamp does not include timezone information already (e.g. default Syslog timestamps). It\'s optional otherwise.\nAcceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00").', - example: '', - footnote: '', - group: 2, - level: 'extended', - name: 'event.timezone', - required: false, - type: 'keyword', - }, - 'event.type': { - description: 'Reserved for future usage.\nPlease avoid using this field for user data.', - example: '', - footnote: '', - group: 2, - level: 'core', - name: 'event.type', - required: false, - type: 'keyword', - }, - }, - group: 2, - name: 'event', - title: 'Event', - type: 'group', - }, - file: { - description: - 'A file is defined as a set of information that has been created on, or has existed on a filesystem. File objects can be associated with host events, network events, and/or file events (e.g., those produced by File Integrity Monitoring [FIM] products or services). File fields provide details about the affected file associated with the event or metric.\n', - fields: { - 'file.ctime': { - description: 'Last time file metadata changed.', - example: '', - footnote: '', + 'An observer is defined as a special network, security, or application\ndevice used to detect, observe, or create network, security, or application-related\nevents and metrics.\n\nThis could be a custom hardware appliance or a server that has been configured\nto run special network, security, or application software. Examples include\nfirewalls, web proxies, intrusion detection/prevention systems, network monitoring\nsensors, web application firewalls, data loss prevention systems, and APM servers.\nThe observer.* fields shall be populated with details of the system, if any,\nthat detects, observes and/or creates a network, security, or application event\nor metric. Message queues and ETL components used in processing events or metrics\nare not considered observers in ECS.', + type: 'group', + fields: [ + { + name: 'egress', + level: 'extended', + type: 'object', + object_type: 'keyword', + description: + 'Observer.egress holds information like interface number and name,\nvlan, and zone information to classify egress traffic. Single armed monitoring\nsuch as a network sensor on a span port should only use observer.ingress\nto categorize traffic.', + default_field: false, + }, + { + name: 'egress.interface.alias', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Interface alias as reported by the system, typically used in firewall\nimplementations for e.g. inside, outside, or dmz logical interface naming.', + example: 'outside', + default_field: false, + }, + { + name: 'egress.interface.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Interface ID as reported by an observer (typically SNMP interface\nID).', + example: 10, + default_field: false, + }, + { + name: 'egress.interface.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Interface name as reported by the system.', + example: 'eth0', + default_field: false, + }, + { + name: 'egress.vlan.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'VLAN ID as reported by the observer.', + example: 10, + default_field: false, + }, + { + name: 'egress.vlan.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Optional VLAN name as reported by the observer.', + example: 'outside', + default_field: false, + }, + { + name: 'egress.zone', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Network zone of outbound traffic as reported by the observer to\ncategorize the destination area of egress traffic, e.g. Internal, External,\nDMZ, HR, Legal, etc.', + example: 'Public_Internet', + default_field: false, + }, + { + name: 'geo.city_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'City name.', + example: 'Montreal', + }, + { + name: 'geo.continent_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the continent.', + example: 'North America', + }, + { + name: 'geo.country_iso_code', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Country ISO code.', + example: 'CA', + }, + { + name: 'geo.country_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Country name.', + example: 'Canada', + }, + { + name: 'geo.location', + level: 'core', + type: 'geo_point', + description: 'Longitude and latitude.', + example: '{ "lon": -73.614830, "lat": 45.505918 }', + }, + { + name: 'geo.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'User-defined description of a location, at the level of granularity\nthey care about.\n\nCould be the name of their data centers, the floor number, if this describes\na local physical entity, city names.\n\nNot typically used in automated geolocation.', + example: 'boston-dc', + }, + { + name: 'geo.region_iso_code', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Region ISO code.', + example: 'CA-QC', + }, + { + name: 'geo.region_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Region name.', + example: 'Quebec', + }, + { + name: 'hostname', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Hostname of the observer.', + }, + { + name: 'ingress', + level: 'extended', + type: 'object', + object_type: 'keyword', + description: + 'Observer.ingress holds information like interface number and name,\nvlan, and zone information to classify ingress traffic. Single armed monitoring\nsuch as a network sensor on a span port should only use observer.ingress\nto categorize traffic.', + default_field: false, + }, + { + name: 'ingress.interface.alias', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Interface alias as reported by the system, typically used in firewall\nimplementations for e.g. inside, outside, or dmz logical interface naming.', + example: 'outside', + default_field: false, + }, + { + name: 'ingress.interface.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Interface ID as reported by an observer (typically SNMP interface\nID).', + example: 10, + default_field: false, + }, + { + name: 'ingress.interface.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Interface name as reported by the system.', + example: 'eth0', + default_field: false, + }, + { + name: 'ingress.vlan.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'VLAN ID as reported by the observer.', + example: 10, + default_field: false, + }, + { + name: 'ingress.vlan.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Optional VLAN name as reported by the observer.', + example: 'outside', + default_field: false, + }, + { + name: 'ingress.zone', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Network zone of incoming traffic as reported by the observer to\ncategorize the source area of ingress traffic. e.g. internal, External, DMZ,\nHR, Legal, etc.', + example: 'DMZ', + default_field: false, + }, + { + name: 'ip', + level: 'core', + type: 'ip', + description: 'IP addresses of the observer.', + }, + { + name: 'mac', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'MAC addresses of the observer', + }, + { + name: 'name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Custom name of the observer.\n\nThis is a name that can be given to an observer. This can be helpful for example\nif multiple firewalls of the same model are used in an organization.\n\nIf no custom name is needed, the field can be left empty.', + example: '1_proxySG', + }, + { + name: 'os.family', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'OS family (such as redhat, debian, freebsd, windows).', + example: 'debian', + }, + { + name: 'os.full', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Operating system name, including the version or code name.', + example: 'Mac OS Mojave', + }, + { + name: 'os.kernel', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Operating system kernel version as a raw string.', + example: '4.4.0-112-generic', + }, + { + name: 'os.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Operating system name, without the version.', + example: 'Mac OS X', + }, + { + name: 'os.platform', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Operating system platform (such centos, ubuntu, windows).', + example: 'darwin', + }, + { + name: 'os.version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Operating system version as a raw string.', + example: '10.14.1', + }, + { + name: 'product', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'The product name of the observer.', + example: 's200', + }, + { + name: 'serial_number', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Observer serial number.', + }, + { + name: 'type', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'The type of the observer the data is coming from.\n\nThere is no predefined list of observer types. Some examples are `forwarder`,\n`firewall`, `ids`, `ips`, `proxy`, `poller`, `sensor`, `APM server`.', + example: 'firewall', + }, + { + name: 'vendor', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Vendor name of the observer.', + example: 'Symantec', + }, + { + name: 'version', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Observer version.', + }, + ], + }, + { + name: 'organization', + title: 'Organization', group: 2, - level: 'extended', - name: 'file.ctime', - required: false, - type: 'date', - }, - 'file.device': { - description: 'Device that is the source of the file.', - example: '', - footnote: '', - group: 2, - level: 'extended', - name: 'file.device', - required: false, - type: 'keyword', - }, - 'file.extension': { - description: 'File extension.\nThis should allow easy filtering by file extensions.', - example: 'png', - footnote: '', - group: 2, - level: 'extended', - name: 'file.extension', - required: false, - type: 'keyword', - }, - 'file.gid': { - description: 'Primary group ID (GID) of the file.', - example: '', - footnote: '', - group: 2, - level: 'extended', - name: 'file.gid', - required: false, - type: 'keyword', - }, - 'file.group': { - description: 'Primary group name of the file.', - example: '', - footnote: '', - group: 2, - level: 'extended', - name: 'file.group', - required: false, - type: 'keyword', - }, - 'file.inode': { - description: 'Inode representing the file in the filesystem.', - example: '', - footnote: '', + description: + 'The organization fields enrich data with information about the company\nor entity the data is associated with.\n\nThese fields help you arrange or filter data stored in an index by one or multiple\norganizations.', + type: 'group', + fields: [ + { + name: 'id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifier for the organization.', + }, + { + name: 'name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Organization name.', + }, + ], + }, + { + name: 'os', + title: 'Operating System', + group: 2, + description: 'The OS fields contain information about the operating system.', + type: 'group', + fields: [ + { + name: 'family', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'OS family (such as redhat, debian, freebsd, windows).', + example: 'debian', + }, + { + name: 'full', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Operating system name, including the version or code name.', + example: 'Mac OS Mojave', + }, + { + name: 'kernel', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Operating system kernel version as a raw string.', + example: '4.4.0-112-generic', + }, + { + name: 'name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Operating system name, without the version.', + example: 'Mac OS X', + }, + { + name: 'platform', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Operating system platform (such centos, ubuntu, windows).', + example: 'darwin', + }, + { + name: 'version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Operating system version as a raw string.', + example: '10.14.1', + }, + ], + }, + { + name: 'package', + title: 'Package', group: 2, - level: 'extended', - name: 'file.inode', - required: false, - type: 'keyword', - }, - 'file.mode': { - description: 'Mode of the file in octal representation.', - example: '416', - footnote: '', + description: + 'These fields contain information about an installed software package.\nIt contains general information about a package, such as name, version or size.\nIt also contains installation details, such as time or location.', + type: 'group', + fields: [ + { + name: 'architecture', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Package architecture.', + example: 'x86_64', + }, + { + name: 'build_version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Additional information about the build version of the installed\npackage.\n\nFor example use the commit SHA of a non-released package.', + example: '36f4f7e89dd61b0988b12ee000b98966867710cd', + default_field: false, + }, + { + name: 'checksum', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Checksum of the installed package for verification.', + example: '68b329da9893e34099c7d8ad5cb9c940', + }, + { + name: 'description', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Description of the package.', + example: + 'Open source programming language to build simple/reliable/efficient\nsoftware.', + }, + { + name: 'install_scope', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Indicating how the package was installed, e.g. user-local, global.', + example: 'global', + }, + { + name: 'installed', + level: 'extended', + type: 'date', + description: 'Time when package was installed.', + }, + { + name: 'license', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'License under which the package was released.\n\nUse a short name, e.g. the license identifier from SPDX License List where\npossible (https://spdx.org/licenses/).', + example: 'Apache License 2.0', + }, + { + name: 'name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Package name', + example: 'go', + }, + { + name: 'path', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Path where the package is installed.', + example: '/usr/local/Cellar/go/1.12.9/', + }, + { + name: 'reference', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Home page or reference URL of the software in this package, if\navailable.', + example: 'https://golang.org', + default_field: false, + }, + { + name: 'size', + level: 'extended', + type: 'long', + format: 'string', + description: 'Package size in bytes.', + example: 62231, + }, + { + name: 'type', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Type of package.\n\nThis should contain the package file type, rather than the package manager\nname. Examples: rpm, dpkg, brew, npm, gem, nupkg, jar.', + example: 'rpm', + default_field: false, + }, + { + name: 'version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Package version', + example: '1.12.9', + }, + ], + }, + { + name: 'pe', + title: 'PE Header', + group: 2, + description: 'These fields contain Windows Portable Executable (PE) metadata.', + type: 'group', + fields: [ + { + name: 'architecture', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'CPU architecture target for the file.', + example: 'x64', + default_field: false, + }, + { + name: 'company', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal company name of the file, provided at compile-time.', + example: 'Microsoft Corporation', + default_field: false, + }, + { + name: 'description', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal description of the file, provided at compile-time.', + example: 'Paint', + default_field: false, + }, + { + name: 'file_version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal version of the file, provided at compile-time.', + example: '6.3.9600.17415', + default_field: false, + }, + { + name: 'imphash', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'A hash of the imports in a PE file. An imphash -- or import hash\n-- can be used to fingerprint binaries even after recompilation or other code-level\ntransformations have occurred, which would change more traditional hash values.\n\nLearn more at https://www.fireeye.com/blog/threat-research/2014/01/tracking-malware-import-hashing.html.', + example: '0c6803c4e922103c4dca5963aad36ddf', + default_field: false, + }, + { + name: 'original_file_name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal name of the file, provided at compile-time.', + example: 'MSPAINT.EXE', + default_field: false, + }, + { + name: 'product', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal product name of the file, provided at compile-time.', + example: 'Microsoft® Windows® Operating System', + default_field: false, + }, + ], + }, + { + name: 'process', + title: 'Process', group: 2, - level: 'extended', - name: 'file.mode', - required: false, - type: 'keyword', - }, - 'file.mtime': { - description: 'Last time file content was modified.', - example: '', - footnote: '', + description: + 'These fields contain information about a process.\n\nThese fields can help you correlate metrics information with a process id/name\nfrom a log message. The `process.pid` often stays in the metric itself and\nis copied to the global field for correlation.', + type: 'group', + fields: [ + { + name: 'args', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Array of process arguments, starting with the absolute path to\nthe executable.\n\nMay be filtered to protect sensitive information.', + example: ['/usr/bin/ssh', '-l', 'user', '10.0.0.16'], + }, + { + name: 'args_count', + level: 'extended', + type: 'long', + description: + 'Length of the process.args array.\n\nThis field can be useful for querying or performing bucket analysis on how\nmany arguments were provided to start a process. More arguments may be an\nindication of suspicious activity.', + example: 4, + default_field: false, + }, + { + name: 'code_signature.exists', + level: 'core', + type: 'boolean', + description: 'Boolean to capture if a signature is present.', + example: 'true', + default_field: false, + }, + { + name: 'code_signature.status', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Additional information about the certificate status.\n\nThis is useful for logging cryptographic errors with the certificate validity\nor trust status. Leave unpopulated if the validity or trust of the certificate\nwas unchecked.', + example: 'ERROR_UNTRUSTED_ROOT', + default_field: false, + }, + { + name: 'code_signature.subject_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Subject name of the code signer', + example: 'Microsoft Corporation', + default_field: false, + }, + { + name: 'code_signature.trusted', + level: 'extended', + type: 'boolean', + description: + 'Stores the trust status of the certificate chain.\n\nValidating the trust of the certificate chain may be complicated, and this\nfield should only be populated by tools that actively check the status.', + example: 'true', + default_field: false, + }, + { + name: 'code_signature.valid', + level: 'extended', + type: 'boolean', + description: + 'Boolean to capture if the digital signature is verified against\nthe binary content.\n\nLeave unpopulated if a certificate was unchecked.', + example: 'true', + default_field: false, + }, + { + name: 'command_line', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + }, + ], + description: + 'Full command line that started the process, including the absolute\npath to the executable, and all arguments.\n\nSome arguments may be filtered to protect sensitive information.', + example: '/usr/bin/ssh -l user 10.0.0.16', + default_field: false, + }, + { + name: 'entity_id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Unique identifier for the process.\n\nThe implementation of this is specified by the data source, but some examples\nof what could be used here are a process-generated UUID, Sysmon Process GUIDs,\nor a hash of some uniquely identifying components of a process.\n\nConstructing a globally unique identifier is a common practice to mitigate\nPID reuse as well as to identify a specific process over time, across multiple\nmonitored hosts.', + example: 'c2c455d9f99375d', + default_field: false, + }, + { + name: 'executable', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Absolute path to the process executable.', + example: '/usr/bin/ssh', + }, + { + name: 'exit_code', + level: 'extended', + type: 'long', + description: + 'The exit code of the process, if this is a termination event.\n\nThe field should be absent if there is no exit code for the event (e.g. process\nstart).', + example: 137, + default_field: false, + }, + { + name: 'hash.md5', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'MD5 hash.', + }, + { + name: 'hash.sha1', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'SHA1 hash.', + }, + { + name: 'hash.sha256', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'SHA256 hash.', + }, + { + name: 'hash.sha512', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'SHA512 hash.', + }, + { + name: 'name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Process name.\n\nSometimes called program name or similar.', + example: 'ssh', + }, + { + name: 'parent.args', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Array of process arguments.\n\nMay be filtered to protect sensitive information.', + example: ['ssh', '-l', 'user', '10.0.0.16'], + default_field: false, + }, + { + name: 'parent.args_count', + level: 'extended', + type: 'long', + description: + 'Length of the process.args array.\n\nThis field can be useful for querying or performing bucket analysis on how\nmany arguments were provided to start a process. More arguments may be an\nindication of suspicious activity.', + example: 4, + default_field: false, + }, + { + name: 'parent.code_signature.exists', + level: 'core', + type: 'boolean', + description: 'Boolean to capture if a signature is present.', + example: 'true', + default_field: false, + }, + { + name: 'parent.code_signature.status', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Additional information about the certificate status.\n\nThis is useful for logging cryptographic errors with the certificate validity\nor trust status. Leave unpopulated if the validity or trust of the certificate\nwas unchecked.', + example: 'ERROR_UNTRUSTED_ROOT', + default_field: false, + }, + { + name: 'parent.code_signature.subject_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Subject name of the code signer', + example: 'Microsoft Corporation', + default_field: false, + }, + { + name: 'parent.code_signature.trusted', + level: 'extended', + type: 'boolean', + description: + 'Stores the trust status of the certificate chain.\n\nValidating the trust of the certificate chain may be complicated, and this\nfield should only be populated by tools that actively check the status.', + example: 'true', + default_field: false, + }, + { + name: 'parent.code_signature.valid', + level: 'extended', + type: 'boolean', + description: + 'Boolean to capture if the digital signature is verified against\nthe binary content.\n\nLeave unpopulated if a certificate was unchecked.', + example: 'true', + default_field: false, + }, + { + name: 'parent.command_line', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + }, + ], + description: + 'Full command line that started the process, including the absolute\npath to the executable, and all arguments.\n\nSome arguments may be filtered to protect sensitive information.', + example: '/usr/bin/ssh -l user 10.0.0.16', + default_field: false, + }, + { + name: 'parent.entity_id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Unique identifier for the process.\n\nThe implementation of this is specified by the data source, but some examples\nof what could be used here are a process-generated UUID, Sysmon Process GUIDs,\nor a hash of some uniquely identifying components of a process.\n\nConstructing a globally unique identifier is a common practice to mitigate\nPID reuse as well as to identify a specific process over time, across multiple\nmonitored hosts.', + example: 'c2c455d9f99375d', + default_field: false, + }, + { + name: 'parent.executable', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + }, + ], + description: 'Absolute path to the process executable.', + example: '/usr/bin/ssh', + default_field: false, + }, + { + name: 'parent.exit_code', + level: 'extended', + type: 'long', + description: + 'The exit code of the process, if this is a termination event.\n\nThe field should be absent if there is no exit code for the event (e.g. process\nstart).', + example: 137, + default_field: false, + }, + { + name: 'parent.hash.md5', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'MD5 hash.', + default_field: false, + }, + { + name: 'parent.hash.sha1', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'SHA1 hash.', + default_field: false, + }, + { + name: 'parent.hash.sha256', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'SHA256 hash.', + default_field: false, + }, + { + name: 'parent.hash.sha512', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'SHA512 hash.', + default_field: false, + }, + { + name: 'parent.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + }, + ], + description: 'Process name.\n\nSometimes called program name or similar.', + example: 'ssh', + default_field: false, + }, + { + name: 'parent.pgid', + level: 'extended', + type: 'long', + format: 'string', + description: 'Identifier of the group of processes the process belongs to.', + default_field: false, + }, + { + name: 'parent.pid', + level: 'core', + type: 'long', + format: 'string', + description: 'Process id.', + example: 4242, + default_field: false, + }, + { + name: 'parent.ppid', + level: 'extended', + type: 'long', + format: 'string', + description: "Parent process' pid.", + example: 4241, + default_field: false, + }, + { + name: 'parent.start', + level: 'extended', + type: 'date', + description: 'The time the process started.', + example: '2016-05-23T08:05:34.853Z', + default_field: false, + }, + { + name: 'parent.thread.id', + level: 'extended', + type: 'long', + format: 'string', + description: 'Thread ID.', + example: 4242, + default_field: false, + }, + { + name: 'parent.thread.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Thread name.', + example: 'thread-0', + default_field: false, + }, + { + name: 'parent.title', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + }, + ], + description: + 'Process title.\n\nThe proctitle, some times the same as process name. Can also be different:\nfor example a browser setting its title to the web page currently opened.', + default_field: false, + }, + { + name: 'parent.uptime', + level: 'extended', + type: 'long', + description: 'Seconds the process has been up.', + example: 1325, + default_field: false, + }, + { + name: 'parent.working_directory', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + }, + ], + description: 'The working directory of the process.', + example: '/home/alice', + default_field: false, + }, + { + name: 'pe.architecture', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'CPU architecture target for the file.', + example: 'x64', + default_field: false, + }, + { + name: 'pe.company', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal company name of the file, provided at compile-time.', + example: 'Microsoft Corporation', + default_field: false, + }, + { + name: 'pe.description', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal description of the file, provided at compile-time.', + example: 'Paint', + default_field: false, + }, + { + name: 'pe.file_version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal version of the file, provided at compile-time.', + example: '6.3.9600.17415', + default_field: false, + }, + { + name: 'pe.imphash', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'A hash of the imports in a PE file. An imphash -- or import hash\n-- can be used to fingerprint binaries even after recompilation or other code-level\ntransformations have occurred, which would change more traditional hash values.\n\nLearn more at https://www.fireeye.com/blog/threat-research/2014/01/tracking-malware-import-hashing.html.', + example: '0c6803c4e922103c4dca5963aad36ddf', + default_field: false, + }, + { + name: 'pe.original_file_name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal name of the file, provided at compile-time.', + example: 'MSPAINT.EXE', + default_field: false, + }, + { + name: 'pe.product', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal product name of the file, provided at compile-time.', + example: 'Microsoft® Windows® Operating System', + default_field: false, + }, + { + name: 'pgid', + level: 'extended', + type: 'long', + format: 'string', + description: 'Identifier of the group of processes the process belongs to.', + }, + { + name: 'pid', + level: 'core', + type: 'long', + format: 'string', + description: 'Process id.', + example: 4242, + }, + { + name: 'ppid', + level: 'extended', + type: 'long', + format: 'string', + description: "Parent process' pid.", + example: 4241, + }, + { + name: 'start', + level: 'extended', + type: 'date', + description: 'The time the process started.', + example: '2016-05-23T08:05:34.853Z', + }, + { + name: 'thread.id', + level: 'extended', + type: 'long', + format: 'string', + description: 'Thread ID.', + example: 4242, + }, + { + name: 'thread.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Thread name.', + example: 'thread-0', + }, + { + name: 'title', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: + 'Process title.\n\nThe proctitle, some times the same as process name. Can also be different:\nfor example a browser setting its title to the web page currently opened.', + }, + { + name: 'uptime', + level: 'extended', + type: 'long', + description: 'Seconds the process has been up.', + example: 1325, + }, + { + name: 'working_directory', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'The working directory of the process.', + example: '/home/alice', + }, + ], + }, + { + name: 'registry', + title: 'Registry', + group: 2, + description: 'Fields related to Windows Registry operations.', + type: 'group', + fields: [ + { + name: 'data.bytes', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Original bytes written with base64 encoding.\n\nFor Windows registry operations, such as SetValueEx and RegQueryValueEx, this\ncorresponds to the data pointed by `lp_data`. This is optional but provides\nbetter recoverability and should be populated for REG_BINARY encoded values.', + example: 'ZQBuAC0AVQBTAAAAZQBuAAAAAAA=', + default_field: false, + }, + { + name: 'data.strings', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'Content when writing string types.\n\nPopulated as an array when writing string data to the registry. For single\nstring registry types (REG_SZ, REG_EXPAND_SZ), this should be an array with\none string. For sequences of string with REG_MULTI_SZ, this array will be\nvariable length. For numeric data, such as REG_DWORD and REG_QWORD, this should\nbe populated with the decimal representation (e.g `"1"`).', + example: '["C:\\rta\\red_ttp\\bin\\myapp.exe"]', + default_field: false, + }, + { + name: 'data.type', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Standard registry type for encoding contents', + example: 'REG_SZ', + default_field: false, + }, + { + name: 'hive', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Abbreviated name for the hive.', + example: 'HKLM', + default_field: false, + }, + { + name: 'key', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Hive-relative path of keys.', + example: + 'SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Image File Execution Options\\winword.exe', + default_field: false, + }, + { + name: 'path', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Full path, including hive, key and value', + example: + 'HKLM\\SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Image File Execution\nOptions\\winword.exe\\Debugger', + default_field: false, + }, + { + name: 'value', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the value written.', + example: 'Debugger', + default_field: false, + }, + ], + }, + { + name: 'related', + title: 'Related', group: 2, - level: 'extended', - name: 'file.mtime', - required: false, - type: 'date', - }, - 'file.owner': { - description: "File owner's username.", - example: '', - footnote: '', + description: + 'This field set is meant to facilitate pivoting around a piece of\ndata.\n\nSome pieces of information can be seen in many places in an ECS event. To facilitate\nsearching for them, store an array of all seen values to their corresponding\nfield in `related.`.\n\nA concrete example is IP addresses, which can be under host, observer, source,\ndestination, client, server, and network.forwarded_ip. If you append all IPs\nto `related.ip`, you can then search for a given IP trivially, no matter where\nit appeared, by querying `related.ip:192.0.2.15`.', + type: 'group', + fields: [ + { + name: 'hash', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + "All the hashes seen on your event. Populating this field, then\nusing it to search for hashes can help in situations where you're unsure what\nthe hash algorithm is (and therefore which key name to search).", + default_field: false, + }, + { + name: 'ip', + level: 'extended', + type: 'ip', + description: 'All of the IPs seen on your event.', + }, + { + name: 'user', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'All the user names seen on your event.', + default_field: false, + }, + ], + }, + { + name: 'rule', + title: 'Rule', group: 2, - level: 'extended', - name: 'file.owner', - required: false, - type: 'keyword', - }, - 'file.path': { - description: 'Path to the file.', - example: '', - footnote: '', + description: + 'Rule fields are used to capture the specifics of any observer or\nagent rules that generate alerts or other notable events.\n\nExamples of data sources that would populate the rule fields include: network\nadmission control platforms, network or host IDS/IPS, network firewalls, web\napplication firewalls, url filters, endpoint detection and response (EDR) systems,\netc.', + type: 'group', + fields: [ + { + name: 'author', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name, organization, or pseudonym of the author or authors who created\nthe rule used to generate this event.', + example: ['Star-Lord'], + default_field: false, + }, + { + name: 'category', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'A categorization value keyword used by the entity using the rule\nfor detection of this event.', + example: 'Attempted Information Leak', + default_field: false, + }, + { + name: 'description', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'The description of the rule generating the event.', + example: 'Block requests to public DNS over HTTPS / TLS protocols', + default_field: false, + }, + { + name: 'id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'A rule ID that is unique within the scope of an agent, observer,\nor other entity using the rule for detection of this event.', + example: 101, + default_field: false, + }, + { + name: 'license', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the license under which the rule used to generate this\nevent is made available.', + example: 'Apache 2.0', + default_field: false, + }, + { + name: 'name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'The name of the rule or signature generating the event.', + example: 'BLOCK_DNS_over_TLS', + default_field: false, + }, + { + name: 'reference', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Reference URL to additional information about the rule used to\ngenerate this event.\n\nThe URL can point to the vendor documentation about the rule. If that is\nnot available, it can also be a link to a more general page describing this\ntype of alert.', + example: 'https://en.wikipedia.org/wiki/DNS_over_TLS', + default_field: false, + }, + { + name: 'ruleset', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the ruleset, policy, group, or parent category in which\nthe rule used to generate this event is a member.', + example: 'Standard_Protocol_Filters', + default_field: false, + }, + { + name: 'uuid', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'A rule ID that is unique within the scope of a set or group of\nagents, observers, or other entities using the rule for detection of this\nevent.', + example: 1100110011, + default_field: false, + }, + { + name: 'version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'The version / revision of the rule being used for analysis.', + example: 1.1, + default_field: false, + }, + ], + }, + { + name: 'search', + title: 'Search', group: 2, - level: 'extended', - name: 'file.path', - required: false, - type: 'keyword', - }, - 'file.size': { - description: 'File size in bytes (field is only added when `type` is `file`).', - example: '', - footnote: '', + description: + 'The Search fields describe information about a search request event:\nquery or pagination. The fields that should be used with this field set include:\n`event.action` to describe the search action (e.g. `search.query`, `search.page`,\netc.), `event.duration` to describe the duration of a search request, `@timestamp`\nto record the event original timestamp and optionally the `source` fields\nto record context information such as `user.id` or `geo`.', + type: 'group', + fields: [ + { + name: 'query.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'An opaque query identifier. This identifier needs to be unique\nto a user query, and all subsequent events (pagination, clicks) need to have\nthe same query identifier.', + example: '2dc15175-de0d-44db-86d8-8a99f41b7a11', + default_field: false, + }, + { + name: 'query.page', + level: 'extended', + type: 'long', + description: + 'For search results that support pagination, this represents the\ncurrent page being requested. Initial search requests are `1` while subsequent\npage requests are incremental.', + example: 1, + default_field: false, + }, + { + name: 'query.value', + level: 'extended', + type: 'keyword', + ignore_above: 4096, + description: + 'The query string being searched on. This field is not analyzed\nand should not be pre-processed in any way in the event (e.g. normalization\nlist lowercasing). This is useful for search use-cases that use a one- box\nstyle search interface. Other interfaces will have to rely on additional custom\nfields or labels to represent things like filters applied, extra parameters,\nuser context, etc.', + example: 'where does the rain in Spain mainly fall', + default_field: false, + }, + { + name: 'results.ids', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + "A list of opaque document IDs representing the results that were\nshown to the user. This is effectively the impression list and it's size should\nbe equal to `results.size`. This field can be empty when there are no results\nto return.", + example: ['user:82375akja9f', 'issue:2782630'], + default_field: false, + }, + { + name: 'results.size', + level: 'extended', + type: 'long', + description: + 'The size of the result set displayed to the user. This should be\nequivalent to the length of the results in `results.ids`. This is also known\nas the page size or limit.', + example: 10, + default_field: false, + }, + { + name: 'results.total', + level: 'extended', + type: 'long', + description: + 'The total number of matches for this query. This number is always\ngreater than or equal to `results.size`. This is the `hits.total` field in\nthe query response.', + example: 134509, + default_field: false, + }, + ], + }, + { + name: 'server', + title: 'Server', group: 2, - level: 'extended', - name: 'file.size', - required: false, - type: 'long', - }, - 'file.target_path': { - description: 'Target path for symlinks.', - example: '', - footnote: '', + description: + 'A Server is defined as the responder in a network connection for\nevents regarding sessions, connections, or bidirectional flow records.\n\nFor TCP events, the server is the receiver of the initial SYN packet(s) of the\nTCP connection. For other protocols, the server is generally the responder in\nthe network transaction. Some systems actually use the term "responder" to refer\nthe server in TCP connections. The server fields describe details about the\nsystem acting as the server in the network event. Server fields are usually\npopulated in conjunction with client fields. Server fields are generally not\npopulated for packet-level events.\n\nClient / server representations can add semantic context to an exchange, which\nis helpful to visualize the data in certain situations. If your context falls\nin that category, you should still ensure that source and destination are filled\nappropriately.', + type: 'group', + fields: [ + { + name: 'address', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Some event server addresses are defined ambiguously. The event\nwill sometimes list an IP, a domain or a unix socket. You should always store\nthe raw address in the `.address` field.\n\nThen it should be duplicated to `.ip` or `.domain`, depending on which one\nit is.', + }, + { + name: 'as.number', + level: 'extended', + type: 'long', + description: + 'Unique number allocated to the autonomous system. The autonomous\nsystem number (ASN) uniquely identifies each network on the Internet.', + example: 15169, + }, + { + name: 'as.organization.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Organization name.', + example: 'Google LLC', + }, + { + name: 'bytes', + level: 'core', + type: 'long', + format: 'bytes', + description: 'Bytes sent from the server to the client.', + example: 184, + }, + { + name: 'domain', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Server domain.', + }, + { + name: 'geo.city_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'City name.', + example: 'Montreal', + }, + { + name: 'geo.continent_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the continent.', + example: 'North America', + }, + { + name: 'geo.country_iso_code', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Country ISO code.', + example: 'CA', + }, + { + name: 'geo.country_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Country name.', + example: 'Canada', + }, + { + name: 'geo.location', + level: 'core', + type: 'geo_point', + description: 'Longitude and latitude.', + example: '{ "lon": -73.614830, "lat": 45.505918 }', + }, + { + name: 'geo.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'User-defined description of a location, at the level of granularity\nthey care about.\n\nCould be the name of their data centers, the floor number, if this describes\na local physical entity, city names.\n\nNot typically used in automated geolocation.', + example: 'boston-dc', + }, + { + name: 'geo.region_iso_code', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Region ISO code.', + example: 'CA-QC', + }, + { + name: 'geo.region_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Region name.', + example: 'Quebec', + }, + { + name: 'ip', + level: 'core', + type: 'ip', + description: 'IP address of the server (IPv4 or IPv6).', + }, + { + name: 'mac', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'MAC address of the server.', + }, + { + name: 'nat.ip', + level: 'extended', + type: 'ip', + description: + 'Translated ip of destination based NAT sessions (e.g. internet\nto private DMZ)\n\nTypically used with load balancers, firewalls, or routers.', + }, + { + name: 'nat.port', + level: 'extended', + type: 'long', + format: 'string', + description: + 'Translated port of destination based NAT sessions (e.g. internet\nto private DMZ)\n\nTypically used with load balancers, firewalls, or routers.', + }, + { + name: 'packets', + level: 'core', + type: 'long', + description: 'Packets sent from the server to the client.', + example: 12, + }, + { + name: 'port', + level: 'core', + type: 'long', + format: 'string', + description: 'Port of the server.', + }, + { + name: 'registered_domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The highest registered server domain, stripped of the subdomain.\n\nFor example, the registered domain for "foo.google.com" is "google.com".\n\nThis value can be determined precisely with a list like the public suffix\nlist (http://publicsuffix.org). Trying to approximate this by simply taking\nthe last two labels will not work well for TLDs such as "co.uk".', + example: 'google.com', + }, + { + name: 'top_level_domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The effective top level domain (eTLD), also known as the domain\nsuffix, is the last part of the domain name. For example, the top level domain\nfor google.com is "com".\n\nThis value can be determined precisely with a list like the public suffix\nlist (http://publicsuffix.org). Trying to approximate this by simply taking\nthe last label will not work well for effective TLDs such as "co.uk".', + example: 'co.uk', + }, + { + name: 'user.domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the directory the user is a member of.\n\nFor example, an LDAP or Active Directory domain name.', + }, + { + name: 'user.email', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'User email address.', + }, + { + name: 'user.full_name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: "User's full name, if available.", + example: 'Albert Einstein', + }, + { + name: 'user.group.domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the directory the group is a member of.\n\nFor example, an LDAP or Active Directory domain name.', + }, + { + name: 'user.group.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifier for the group on the system/platform.', + }, + { + name: 'user.group.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the group.', + }, + { + name: 'user.hash', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Unique user hash to correlate information for a user in anonymized\nform.\n\nUseful if `user.id` or `user.name` contain confidential information and cannot\nbe used.', + }, + { + name: 'user.id', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifier of the user.', + }, + { + name: 'user.name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Short name or login of the user.', + example: 'albert', + }, + ], + }, + { + name: 'service', + title: 'Service', group: 2, - level: 'extended', - name: 'file.target_path', - required: false, - type: 'keyword', - }, - 'file.type': { - description: 'File type (file, dir, or symlink).', - example: '', - footnote: '', + description: + 'The service fields describe the service for or from which the data\nwas collected.\n\nThese fields help you find and correlate logs for a specific service and version.', + type: 'group', + fields: [ + { + name: 'ephemeral_id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Ephemeral identifier of this service (if one exists).\n\nThis id normally changes across restarts, but `service.id` does not.', + example: '8a4f500f', + }, + { + name: 'id', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'Unique identifier of the running service. If the service is comprised\nof many nodes, the `service.id` should be the same for all nodes.\n\nThis id should uniquely identify the service. This makes it possible to correlate\nlogs and metrics for one specific service, no matter which particular node\nemitted the event.\n\nNote that if you need to see the events from one specific host of the service,\nyou should filter on that `host.name` or `host.id` instead.', + example: 'd37e5ebfe0ae6c4972dbe9f0174a1637bb8247f6', + }, + { + name: 'name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the service data is collected from.\n\nThe name of the service is normally user given. This allows for distributed\nservices that run on multiple hosts to correlate the related instances based\non the name.\n\nIn the case of Elasticsearch the `service.name` could contain the cluster\nname. For Beats the `service.name` is by default a copy of the `service.type`\nfield if no name is specified.', + example: 'elasticsearch-metrics', + }, + { + name: 'node.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of a service node.\n\nThis allows for two nodes of the same service running on the same host to\nbe differentiated. Therefore, `service.node.name` should typically be unique\nacross nodes of a given service.\n\nIn the case of Elasticsearch, the `service.node.name` could contain the unique\nnode name within the Elasticsearch cluster. In cases where the service does not\nhave the concept of a node name, the host name or container name can be used\nto distinguish running instances that make up this service. If those do not\nprovide uniqueness (e.g. multiple instances of the service running on the\nsame host) - the node name can be manually set.', + example: 'instance-0000000016', + }, + { + name: 'state', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Current state of the service.', + }, + { + name: 'type', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'The type of the service data is collected from.\n\nThe type can be used to group and correlate logs and metrics from one service\ntype.\n\nExample: If logs or metrics are collected from Elasticsearch, `service.type`\nwould be `elasticsearch`.', + example: 'elasticsearch', + }, + { + name: 'version', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'Version of the service the data was collected from.\n\nThis allows to look at a data set only for a specific version of a service.', + example: '3.2.4', + }, + ], + }, + { + name: 'source', + title: 'Source', group: 2, - level: 'extended', - name: 'file.type', - required: false, - type: 'keyword', - }, - 'file.uid': { - description: 'The user ID (UID) or security identifier (SID) of the file owner.', - example: '', - footnote: '', + description: + 'Source fields describe details about the source of a packet/event.\n\nSource fields are usually populated in conjunction with destination fields.', + type: 'group', + fields: [ + { + name: 'address', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Some event source addresses are defined ambiguously. The event\nwill sometimes list an IP, a domain or a unix socket. You should always store\nthe raw address in the `.address` field.\n\nThen it should be duplicated to `.ip` or `.domain`, depending on which one\nit is.', + }, + { + name: 'as.number', + level: 'extended', + type: 'long', + description: + 'Unique number allocated to the autonomous system. The autonomous\nsystem number (ASN) uniquely identifies each network on the Internet.', + example: 15169, + }, + { + name: 'as.organization.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Organization name.', + example: 'Google LLC', + }, + { + name: 'bytes', + level: 'core', + type: 'long', + format: 'bytes', + description: 'Bytes sent from the source to the destination.', + example: 184, + }, + { + name: 'domain', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Source domain.', + }, + { + name: 'geo.city_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'City name.', + example: 'Montreal', + }, + { + name: 'geo.continent_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the continent.', + example: 'North America', + }, + { + name: 'geo.country_iso_code', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Country ISO code.', + example: 'CA', + }, + { + name: 'geo.country_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Country name.', + example: 'Canada', + }, + { + name: 'geo.location', + level: 'core', + type: 'geo_point', + description: 'Longitude and latitude.', + example: '{ "lon": -73.614830, "lat": 45.505918 }', + }, + { + name: 'geo.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'User-defined description of a location, at the level of granularity\nthey care about.\n\nCould be the name of their data centers, the floor number, if this describes\na local physical entity, city names.\n\nNot typically used in automated geolocation.', + example: 'boston-dc', + }, + { + name: 'geo.region_iso_code', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Region ISO code.', + example: 'CA-QC', + }, + { + name: 'geo.region_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Region name.', + example: 'Quebec', + }, + { + name: 'ip', + level: 'core', + type: 'ip', + description: 'IP address of the source (IPv4 or IPv6).', + }, + { + name: 'mac', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'MAC address of the source.', + }, + { + name: 'nat.ip', + level: 'extended', + type: 'ip', + description: + 'Translated ip of source based NAT sessions (e.g. internal client\nto internet)\n\nTypically connections traversing load balancers, firewalls, or routers.', + }, + { + name: 'nat.port', + level: 'extended', + type: 'long', + format: 'string', + description: + 'Translated port of source based NAT sessions. (e.g. internal client\nto internet)\n\nTypically used with load balancers, firewalls, or routers.', + }, + { + name: 'packets', + level: 'core', + type: 'long', + description: 'Packets sent from the source to the destination.', + example: 12, + }, + { + name: 'port', + level: 'core', + type: 'long', + format: 'string', + description: 'Port of the source.', + }, + { + name: 'registered_domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The highest registered source domain, stripped of the subdomain.\n\nFor example, the registered domain for "foo.google.com" is "google.com".\n\nThis value can be determined precisely with a list like the public suffix\nlist (http://publicsuffix.org). Trying to approximate this by simply taking\nthe last two labels will not work well for TLDs such as "co.uk".', + example: 'google.com', + }, + { + name: 'top_level_domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The effective top level domain (eTLD), also known as the domain\nsuffix, is the last part of the domain name. For example, the top level domain\nfor google.com is "com".\n\nThis value can be determined precisely with a list like the public suffix\nlist (http://publicsuffix.org). Trying to approximate this by simply taking\nthe last label will not work well for effective TLDs such as "co.uk".', + example: 'co.uk', + }, + { + name: 'user.domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the directory the user is a member of.\n\nFor example, an LDAP or Active Directory domain name.', + }, + { + name: 'user.email', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'User email address.', + }, + { + name: 'user.full_name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: "User's full name, if available.", + example: 'Albert Einstein', + }, + { + name: 'user.group.domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the directory the group is a member of.\n\nFor example, an LDAP or Active Directory domain name.', + }, + { + name: 'user.group.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifier for the group on the system/platform.', + }, + { + name: 'user.group.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the group.', + }, + { + name: 'user.hash', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Unique user hash to correlate information for a user in anonymized\nform.\n\nUseful if `user.id` or `user.name` contain confidential information and cannot\nbe used.', + }, + { + name: 'user.id', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifier of the user.', + }, + { + name: 'user.name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Short name or login of the user.', + example: 'albert', + }, + ], + }, + { + name: 'threat', + title: 'Threat', group: 2, - level: 'extended', - name: 'file.uid', - required: false, - type: 'keyword', - }, - }, - group: 2, - name: 'file', - title: 'File', - type: 'group', - }, - geo: { - description: - 'Geo fields can carry data about a specific location related to an event or geo information derived from an IP field.\n', - fields: { - 'geo.city_name': { - description: 'City name.', - example: 'Montreal', - footnote: '', + description: + 'Fields to classify events and alerts according to a threat taxonomy\nsuch as the Mitre ATT&CK framework.\n\nThese fields are for users to classify alerts from all of their sources (e.g.\nIDS, NGFW, etc.) within a common taxonomy. The threat.tactic.* are meant to\ncapture the high level category of the threat (e.g. "impact"). The threat.technique.*\nfields are meant to capture which kind of approach is used by this detected\nthreat, to accomplish the goal (e.g. "endpoint denial of service").', + type: 'group', + fields: [ + { + name: 'framework', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the threat framework used to further categorize and classify\nthe tactic and technique of the reported threat. Framework classification\ncan be provided by detecting systems, evaluated at ingest time, or retrospectively\ntagged to events.', + example: 'MITRE ATT&CK', + }, + { + name: 'tactic.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The id of tactic used by this threat. You can use the Mitre ATT&CK\nMatrix Tactic categorization, for example. (ex. https://attack.mitre.org/tactics/TA0040/\n)', + example: 'TA0040', + }, + { + name: 'tactic.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the type of tactic used by this threat. You can use the\nMitre ATT&CK Matrix Tactic categorization, for example. (ex. https://attack.mitre.org/tactics/TA0040/\n)', + example: 'impact', + }, + { + name: 'tactic.reference', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The reference url of tactic used by this threat. You can use the\nMitre ATT&CK Matrix Tactic categorization, for example. (ex. https://attack.mitre.org/tactics/TA0040/\n)', + example: 'https://attack.mitre.org/tactics/TA0040/', + }, + { + name: 'technique.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The id of technique used by this tactic. You can use the Mitre\nATT&CK Matrix Tactic categorization, for example. (ex. https://attack.mitre.org/techniques/T1499/\n)', + example: 'T1499', + }, + { + name: 'technique.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: + 'The name of technique used by this tactic. You can use the Mitre\nATT&CK Matrix Tactic categorization, for example. (ex. https://attack.mitre.org/techniques/T1499/\n)', + example: 'endpoint denial of service', + }, + { + name: 'technique.reference', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The reference url of technique used by this tactic. You can use\nthe Mitre ATT&CK Matrix Tactic categorization, for example. (ex. https://attack.mitre.org/techniques/T1499/\n)', + example: 'https://attack.mitre.org/techniques/T1499/', + }, + ], + }, + { + name: 'tls', + title: 'TLS', group: 2, - level: 'core', - name: 'geo.city_name', - required: false, - type: 'keyword', - }, - 'geo.continent_name': { - description: 'Name of the continent.', - example: 'North America', - footnote: '', + description: + 'Fields related to a TLS connection. These fields focus on the TLS\nprotocol itself and intentionally avoids in-depth analysis of the related x.509\ncertificate files.', + type: 'group', + fields: [ + { + name: 'cipher', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'String indicating the cipher used during the current connection.', + example: 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256', + default_field: false, + }, + { + name: 'client.certificate', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'PEM-encoded stand-alone certificate offered by the client. This\nis usually mutually-exclusive of `client.certificate_chain` since this value\nalso exists in that list.', + example: 'MII...', + default_field: false, + }, + { + name: 'client.certificate_chain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Array of PEM-encoded certificates that make up the certificate\nchain offered by the client. This is usually mutually-exclusive of `client.certificate`\nsince that value should be the first certificate in the chain.', + example: ['MII...', 'MII...'], + default_field: false, + }, + { + name: 'client.hash.md5', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Certificate fingerprint using the MD5 digest of DER-encoded version\nof certificate offered by the client. For consistency with other hash values,\nthis value should be formatted as an uppercase hash.', + example: '0F76C7F2C55BFD7D8E8B8F4BFBF0C9EC', + default_field: false, + }, + { + name: 'client.hash.sha1', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Certificate fingerprint using the SHA1 digest of DER-encoded version\nof certificate offered by the client. For consistency with other hash values,\nthis value should be formatted as an uppercase hash.', + example: '9E393D93138888D288266C2D915214D1D1CCEB2A', + default_field: false, + }, + { + name: 'client.hash.sha256', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Certificate fingerprint using the SHA256 digest of DER-encoded\nversion of certificate offered by the client. For consistency with other hash\nvalues, this value should be formatted as an uppercase hash.', + example: '0687F666A054EF17A08E2F2162EAB4CBC0D265E1D7875BE74BF3C712CA92DAF0', + default_field: false, + }, + { + name: 'client.issuer', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Distinguished name of subject of the issuer of the x.509 certificate\npresented by the client.', + example: 'CN=MyDomain Root CA, OU=Infrastructure Team, DC=mydomain, DC=com', + default_field: false, + }, + { + name: 'client.ja3', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'A hash that identifies clients based on how they perform an SSL/TLS\nhandshake.', + example: 'd4e5b18d6b55c71272893221c96ba240', + default_field: false, + }, + { + name: 'client.not_after', + level: 'extended', + type: 'date', + description: + 'Date/Time indicating when client certificate is no longer considered\nvalid.', + example: '2021-01-01T00:00:00.000Z', + default_field: false, + }, + { + name: 'client.not_before', + level: 'extended', + type: 'date', + description: 'Date/Time indicating when client certificate is first considered\nvalid.', + example: '1970-01-01T00:00:00.000Z', + default_field: false, + }, + { + name: 'client.server_name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Also called an SNI, this tells the server which hostname to which\nthe client is attempting to connect. When this value is available, it should\nget copied to `destination.domain`.', + example: 'www.elastic.co', + default_field: false, + }, + { + name: 'client.subject', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Distinguished name of subject of the x.509 certificate presented\nby the client.', + example: 'CN=myclient, OU=Documentation Team, DC=mydomain, DC=com', + default_field: false, + }, + { + name: 'client.supported_ciphers', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Array of ciphers offered by the client during the client hello.', + example: [ + 'TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384', + 'TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384', + '...', + ], + default_field: false, + }, + { + name: 'curve', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'String indicating the curve used for the given cipher, when applicable.', + example: 'secp256r1', + default_field: false, + }, + { + name: 'established', + level: 'extended', + type: 'boolean', + description: + 'Boolean flag indicating if the TLS negotiation was successful and\ntransitioned to an encrypted tunnel.', + default_field: false, + }, + { + name: 'next_protocol', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'String indicating the protocol being tunneled. Per the values in\nthe IANA registry (https://www.iana.org/assignments/tls-extensiontype-values/tls-extensiontype-values.xhtml#alpn-protocol-ids),\nthis string should be lower case.', + example: 'http/1.1', + default_field: false, + }, + { + name: 'resumed', + level: 'extended', + type: 'boolean', + description: + 'Boolean flag indicating if this TLS connection was resumed from\nan existing TLS negotiation.', + default_field: false, + }, + { + name: 'server.certificate', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'PEM-encoded stand-alone certificate offered by the server. This\nis usually mutually-exclusive of `server.certificate_chain` since this value\nalso exists in that list.', + example: 'MII...', + default_field: false, + }, + { + name: 'server.certificate_chain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Array of PEM-encoded certificates that make up the certificate\nchain offered by the server. This is usually mutually-exclusive of `server.certificate`\nsince that value should be the first certificate in the chain.', + example: ['MII...', 'MII...'], + default_field: false, + }, + { + name: 'server.hash.md5', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Certificate fingerprint using the MD5 digest of DER-encoded version\nof certificate offered by the server. For consistency with other hash values,\nthis value should be formatted as an uppercase hash.', + example: '0F76C7F2C55BFD7D8E8B8F4BFBF0C9EC', + default_field: false, + }, + { + name: 'server.hash.sha1', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Certificate fingerprint using the SHA1 digest of DER-encoded version\nof certificate offered by the server. For consistency with other hash values,\nthis value should be formatted as an uppercase hash.', + example: '9E393D93138888D288266C2D915214D1D1CCEB2A', + default_field: false, + }, + { + name: 'server.hash.sha256', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Certificate fingerprint using the SHA256 digest of DER-encoded\nversion of certificate offered by the server. For consistency with other hash\nvalues, this value should be formatted as an uppercase hash.', + example: '0687F666A054EF17A08E2F2162EAB4CBC0D265E1D7875BE74BF3C712CA92DAF0', + default_field: false, + }, + { + name: 'server.issuer', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Subject of the issuer of the x.509 certificate presented by the\nserver.', + example: 'CN=MyDomain Root CA, OU=Infrastructure Team, DC=mydomain, DC=com', + default_field: false, + }, + { + name: 'server.ja3s', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'A hash that identifies servers based on how they perform an SSL/TLS\nhandshake.', + example: '394441ab65754e2207b1e1b457b3641d', + default_field: false, + }, + { + name: 'server.not_after', + level: 'extended', + type: 'date', + description: + 'Timestamp indicating when server certificate is no longer considered\nvalid.', + example: '2021-01-01T00:00:00.000Z', + default_field: false, + }, + { + name: 'server.not_before', + level: 'extended', + type: 'date', + description: 'Timestamp indicating when server certificate is first considered\nvalid.', + example: '1970-01-01T00:00:00.000Z', + default_field: false, + }, + { + name: 'server.subject', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Subject of the x.509 certificate presented by the server.', + example: 'CN=www.mydomain.com, OU=Infrastructure Team, DC=mydomain, DC=com', + default_field: false, + }, + { + name: 'version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Numeric part of the version parsed from the original string.', + example: '1.2', + default_field: false, + }, + { + name: 'version_protocol', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Normalized lowercase protocol name parsed from original string.', + example: 'tls', + default_field: false, + }, + ], + }, + { + name: 'tracing', + title: 'Tracing', group: 2, - level: 'core', - name: 'geo.continent_name', - required: false, - type: 'keyword', - }, - 'geo.country_iso_code': { - description: 'Country ISO code.', - example: 'CA', - footnote: '', + description: + 'Distributed tracing makes it possible to analyze performance throughout\na microservice architecture all in one view. This is accomplished by tracing\nall of the requests - from the initial web request in the front-end service\n- to queries made through multiple back-end services.', + type: 'group', + fields: [ + { + name: 'trace.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Unique identifier of the trace.\n\nA trace groups multiple events like transactions that belong together. For\nexample, a user request handled by multiple inter-connected services.', + example: '4bf92f3577b34da6a3ce929d0e0e4736', + }, + { + name: 'transaction.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Unique identifier of the transaction.\n\nA transaction is the highest level of work measured within a service, such\nas a request to a server.', + example: '00f067aa0ba902b7', + }, + ], + }, + { + name: 'url', + title: 'URL', group: 2, - level: 'core', - name: 'geo.country_iso_code', - required: false, - type: 'keyword', - }, - 'geo.country_name': { - description: 'Country name.', - example: 'Canada', - footnote: '', + description: + 'URL fields provide support for complete or partial URLs, and supports\nthe breaking down into scheme, domain, path, and so on.', + type: 'group', + fields: [ + { + name: 'domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Domain of the url, such as "www.elastic.co".\n\nIn some cases a URL may refer to an IP and/or port directly, without a domain\nname. In this case, the IP address would go to the `domain` field.', + example: 'www.elastic.co', + }, + { + name: 'extension', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The field contains the file extension from the original request\nurl.\n\nThe file extension is only set if it exists, as not every url has a file extension.\n\nThe leading period must not be included. For example, the value must be "png",\nnot ".png".', + example: 'png', + }, + { + name: 'fragment', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Portion of the url after the `#`, such as "top".\n\nThe `#` is not part of the fragment.', + }, + { + name: 'full', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: + 'If full URLs are important to your use case, they should be stored\nin `url.full`, whether this field is reconstructed or present in the event\nsource.', + example: 'https://www.elastic.co:443/search?q=elasticsearch#top', + }, + { + name: 'original', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: + 'Unmodified original url as seen in the event source.\n\nNote that in network monitoring, the observed URL may be a full URL, whereas\nin access logs, the URL is often just represented as a path.\n\nThis field is meant to represent the URL as it was observed, complete or not.', + example: + 'https://www.elastic.co:443/search?q=elasticsearch#top or /search?q=elasticsearch', + }, + { + name: 'password', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Password of the request.', + }, + { + name: 'path', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Path of the request, such as "/search".', + }, + { + name: 'port', + level: 'extended', + type: 'long', + format: 'string', + description: 'Port of the request, such as 443.', + example: 443, + }, + { + name: 'query', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The query field describes the query string of the request, such\nas "q=elasticsearch".\n\nThe `?` is excluded from the query string. If a URL contains no `?`, there\nis no query field. If there is a `?` but no query, the query field exists\nwith an empty string. The `exists` query can be used to differentiate between\nthe two cases.', + }, + { + name: 'registered_domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The highest registered url domain, stripped of the subdomain.\n\nFor example, the registered domain for "foo.google.com" is "google.com".\n\nThis value can be determined precisely with a list like the public suffix\nlist (http://publicsuffix.org). Trying to approximate this by simply taking\nthe last two labels will not work well for TLDs such as "co.uk".', + example: 'google.com', + }, + { + name: 'scheme', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Scheme of the request, such as "https".\n\nNote: The `:` is not part of the scheme.', + example: 'https', + }, + { + name: 'top_level_domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The effective top level domain (eTLD), also known as the domain\nsuffix, is the last part of the domain name. For example, the top level domain\nfor google.com is "com".\n\nThis value can be determined precisely with a list like the public suffix\nlist (http://publicsuffix.org). Trying to approximate this by simply taking\nthe last label will not work well for effective TLDs such as "co.uk".', + example: 'co.uk', + }, + { + name: 'username', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Username of the request.', + }, + ], + }, + { + name: 'user', + title: 'User', group: 2, - level: 'core', - name: 'geo.country_name', - required: false, - type: 'keyword', - }, - 'geo.location': { - description: 'Longitude and latitude.', - example: '{ "lon": -73.614830, "lat": 45.505918 }', - footnote: '', + description: + 'The user fields describe information about the user that is relevant\nto the event.\n\nFields can have one entry or multiple entries. If a user has more than one id,\nprovide an array that includes all of them.', + type: 'group', + fields: [ + { + name: 'domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the directory the user is a member of.\n\nFor example, an LDAP or Active Directory domain name.', + }, + { + name: 'email', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'User email address.', + }, + { + name: 'full_name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: "User's full name, if available.", + example: 'Albert Einstein', + }, + { + name: 'group.domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the directory the group is a member of.\n\nFor example, an LDAP or Active Directory domain name.', + }, + { + name: 'group.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifier for the group on the system/platform.', + }, + { + name: 'group.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the group.', + }, + { + name: 'hash', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Unique user hash to correlate information for a user in anonymized\nform.\n\nUseful if `user.id` or `user.name` contain confidential information and cannot\nbe used.', + }, + { + name: 'id', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifier of the user.', + }, + { + name: 'name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Short name or login of the user.', + example: 'albert', + }, + ], + }, + { + name: 'user_agent', + title: 'User agent', group: 2, - level: 'core', - name: 'geo.location', - required: false, - type: 'geo_point', - }, - 'geo.name': { description: - 'User-defined description of a location, at the level of granularity they care about.\nCould be the name of their data centers, the floor number, if this describes a local physical entity, city names.\nNot typically used in automated geolocation.', - example: 'boston-dc', - footnote: '', + 'The user_agent fields normally come from a browser request.\n\nThey often show up in web service logs coming from the parsed user agent string.', + type: 'group', + fields: [ + { + name: 'device.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the device.', + example: 'iPhone', + }, + { + name: 'name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the user agent.', + example: 'Safari', + }, + { + name: 'original', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + }, + ], + description: 'Unparsed user_agent string.', + example: + 'Mozilla/5.0 (iPhone; CPU iPhone OS 12_1 like Mac OS X) AppleWebKit/605.1.15\n(KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1', + }, + { + name: 'os.family', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'OS family (such as redhat, debian, freebsd, windows).', + example: 'debian', + }, + { + name: 'os.full', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Operating system name, including the version or code name.', + example: 'Mac OS Mojave', + }, + { + name: 'os.kernel', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Operating system kernel version as a raw string.', + example: '4.4.0-112-generic', + }, + { + name: 'os.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Operating system name, without the version.', + example: 'Mac OS X', + }, + { + name: 'os.platform', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Operating system platform (such centos, ubuntu, windows).', + example: 'darwin', + }, + { + name: 'os.version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Operating system version as a raw string.', + example: '10.14.1', + }, + { + name: 'version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Version of the user agent.', + example: 12, + }, + ], + }, + { + name: 'vlan', + title: 'VLAN', group: 2, - level: 'extended', - name: 'geo.name', - required: false, - type: 'keyword', - }, - 'geo.region_iso_code': { - description: 'Region ISO code.', - example: 'CA-QC', - footnote: '', + description: + 'The VLAN fields are used to identify 802.1q tag(s) of a packet,\nas well as ingress and egress VLAN associations of an observer in relation to\na specific packet or connection.\n\nNetwork.vlan fields are used to record a single VLAN tag, or the outer tag in\nthe case of q-in-q encapsulations, for a packet or connection as observed, typically\nprovided by a network sensor (e.g. Zeek, Wireshark) passively reporting on traffic.\n\nNetwork.inner VLAN fields are used to report inner q-in-q 802.1q tags (multiple\n802.1q encapsulations) as observed, typically provided by a network sensor (e.g.\nZeek, Wireshark) passively reporting on traffic. Network.inner VLAN fields should\nonly be used in addition to network.vlan fields to indicate q-in-q tagging.\n\nObserver.ingress and observer.egress VLAN values are used to record observer\nspecific information when observer events contain discrete ingress and egress\nVLAN information, typically provided by firewalls, routers, or load balancers.', + type: 'group', + fields: [ + { + name: 'id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'VLAN ID as reported by the observer.', + example: 10, + default_field: false, + }, + { + name: 'name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Optional VLAN name as reported by the observer.', + example: 'outside', + default_field: false, + }, + ], + }, + { + name: 'vulnerability', + title: 'Vulnerability', group: 2, - level: 'core', - name: 'geo.region_iso_code', - required: false, - type: 'keyword', - }, - 'geo.region_name': { - description: 'Region name.', - example: 'Quebec', - footnote: '', - group: 2, - level: 'core', - name: 'geo.region_name', - required: false, - type: 'keyword', - }, - }, - group: 2, - name: 'geo', - title: 'Geo', - type: 'group', - }, - group: { - description: 'The group fields are meant to represent groups that are relevant to the event.\n', - fields: { - 'group.id': { - description: 'Unique identifier for the group on the system/platform.', - example: '', - footnote: '', - group: 2, - level: 'extended', - name: 'group.id', - required: false, - type: 'keyword', - }, - 'group.name': { - description: 'Name of the group.', - example: '', - footnote: '', - group: 2, - level: 'extended', - name: 'group.name', - required: false, - type: 'keyword', - }, - }, - group: 2, - name: 'group', - title: 'Group', - type: 'group', - }, - host: { - description: - 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or on which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.\n', - fields: { - 'host.architecture': { - description: 'Operating system architecture.', - example: 'x86_64', - footnote: '', - group: 2, - level: 'core', - name: 'host.architecture', - required: false, - type: 'keyword', - }, - 'host.hostname': { - description: - 'Hostname of the host.\nIt normally contains what the `hostname` command returns on the host machine.', - example: '', - footnote: '', - group: 2, - level: 'core', - name: 'host.hostname', - required: false, - type: 'keyword', - }, - 'host.id': { - description: - 'Unique host id.\nAs hostname is not always unique, use values that are meaningful in your environment.\nExample: The current usage of `beat.name`.', - example: '', - footnote: '', - group: 2, - level: 'core', - name: 'host.id', - required: false, - type: 'keyword', - }, - 'host.ip': { - description: 'Host ip address.', - example: '', - footnote: '', - group: 2, - level: 'core', - name: 'host.ip', - required: false, - type: 'ip', - }, - 'host.mac': { - description: 'Host mac address.', - example: '', - footnote: '', - group: 2, - level: 'core', - name: 'host.mac', - required: false, - type: 'keyword', - }, - 'host.name': { - description: - 'Name of the host.\nIt can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.', - example: '', - footnote: '', - group: 2, - level: 'core', - name: 'host.name', - required: false, - type: 'keyword', - }, - 'host.type': { - description: - 'Type of host.\nFor Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.', - example: '', - footnote: '', - group: 2, - level: 'core', - name: 'host.type', - required: false, - type: 'keyword', - }, - }, - group: 2, - name: 'host', - title: 'Host', - type: 'group', - }, - http: { - description: 'Fields related to HTTP activity.\n', - fields: { - 'http.request.method': { - description: 'Http request method.', - example: 'GET, POST, PUT', - footnote: '', - group: 2, - level: 'extended', - name: 'http.request.method', - required: false, - type: 'keyword', - }, - 'http.request.referrer': { - description: 'Referrer for this HTTP request.', - example: 'https://blog.example.com/', - footnote: '', - group: 2, - level: 'extended', - name: 'http.request.referrer', - required: false, - type: 'keyword', - }, - 'http.response.body': { - description: 'The full http response body.', - example: 'Hello world', - footnote: '', - group: 2, - level: 'extended', - name: 'http.response.body', - required: false, - type: 'keyword', - }, - 'http.response.status_code': { - description: 'Http response status code.', - example: '404', - footnote: '', - group: 2, - level: 'extended', - name: 'http.response.status_code', - required: false, - type: 'long', - }, - 'http.version': { - description: 'Http version.', - example: '1.1', - footnote: '', - group: 2, - level: 'extended', - name: 'http.version', - required: false, - type: 'keyword', - }, - }, - group: 2, - name: 'http', - title: 'HTTP', - type: 'group', - }, - log: { - description: 'Fields which are specific to log events.\n', - fields: { - 'log.level': { - description: 'Log level of the log event.\nSome examples are `WARN`, `ERR`, `INFO`.', - example: 'ERR', - footnote: '', - group: 2, - level: 'core', - name: 'log.level', - required: false, - type: 'keyword', - }, - 'log.original': { - description: - "This is the original log message and contains the full log message before splitting it up in multiple parts.\nIn contrast to the `message` field which can contain an extracted part of the log message, this field contains the original, full log message. It can have already some modifications applied like encoding or new lines removed to clean up the log message.\nThis field is not indexed and doc_values are disabled so it can't be queried but the value can be retrieved from `_source`.", - example: 'Sep 19 08:26:10 localhost My log', - footnote: '', - group: 2, - level: 'core', - name: 'log.original', - required: false, - type: '(not indexed)', - }, - }, - group: 2, - name: 'log', - title: 'Log', - type: 'group', - }, - network: { - description: - 'The network is defined as the communication path over which a host or network event happens. The network.* fields should be populated with details about the network activity associated with an event.\n', - fields: { - 'network.application': { - description: - 'A name given to an application. This can be arbitrarily assigned for things like microservices, but also apply to things like skype, icq, facebook, twitter. This would be used in situations where the vendor or service can be decoded such as from the source/dest IP owners, ports, or wire format.', - example: 'AIM', - footnote: '', - group: 2, - level: 'extended', - name: 'network.application', - required: false, - type: 'keyword', - }, - 'network.bytes': { - description: - 'Total bytes transferred in both directions.\nIf `source.bytes` and `destination.bytes` are known, `network.bytes` is their sum.', - example: '368', - footnote: '', - group: 2, - level: 'core', - name: 'network.bytes', - required: false, - type: 'long', - }, - 'network.community_id': { - description: - 'A hash of source and destination IPs and ports, as well as the protocol used in a communication. This is a tool-agnostic standard to identify flows.\nLearn more at https://github.com/corelight/community-id-spec.', - example: '1:hO+sN4H+MG5MY/8hIrXPqc4ZQz0=', - footnote: '', - group: 2, - level: 'extended', - name: 'network.community_id', - required: false, - type: 'keyword', - }, - 'network.direction': { - description: - "Direction of the network traffic.\nRecommended values are:\n * inbound\n * outbound\n * internal\n * external\n * unknown\n\nWhen mapping events from a host-based monitoring context, populate this field from the host's point of view.\nWhen mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of your network perimeter.", - example: 'inbound', - footnote: '', - group: 2, - level: 'core', - name: 'network.direction', - required: false, - type: 'keyword', - }, - 'network.forwarded_ip': { - description: 'Host IP address when the source IP address is the proxy.', - example: '192.1.1.2', - footnote: '', - group: 2, - level: 'core', - name: 'network.forwarded_ip', - required: false, - type: 'ip', - }, - 'network.iana_number': { - description: - 'IANA Protocol Number (https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml). Standardized list of protocols. This aligns well with NetFlow and sFlow related logs which use the IANA Protocol Number.', - example: '6', - footnote: '', - group: 2, - level: 'extended', - name: 'network.iana_number', - required: false, - type: 'keyword', - }, - 'network.name': { - description: 'Name given by operators to sections of their network.', - example: 'Guest Wifi', - footnote: '', - group: 2, - level: 'extended', - name: 'network.name', - required: false, - type: 'keyword', - }, - 'network.packets': { - description: - 'Total packets transferred in both directions.\nIf `source.packets` and `destination.packets` are known, `network.packets` is their sum.', - example: '24', - footnote: '', - group: 2, - level: 'core', - name: 'network.packets', - required: false, - type: 'long', - }, - 'network.protocol': { - description: 'L7 Network protocol name. ex. http, lumberjack, transport protocol', - example: 'http', - footnote: '', - group: 2, - level: 'core', - name: 'network.protocol', - required: false, - type: 'keyword', - }, - 'network.transport': { - description: - 'Same as network.iana_number, but instead using the Keyword name of the transport layer (UDP, TCP, IPv6-ICMP, etc.)', - example: 'TCP', - footnote: '', - group: 2, - level: 'core', - name: 'network.transport', - required: false, - type: 'keyword', - }, - 'network.type': { - description: - 'In the OSI Model this would be the Network Layer. IPv4, IPv6, IPSec, PIM, etc', - example: 'IPv4', - footnote: '', - group: 2, - level: 'core', - name: 'network.type', - required: false, - type: 'keyword', - }, - }, - group: 2, - name: 'network', - title: 'Network', - type: 'group', - }, - observer: { - description: - 'An observer is defined as a special network, security, or application device used to detect, observe, or create network, security, or application-related events and metrics. This could be a custom hardware appliance or a server that has been configured to run special network, security, or application software. Examples include firewalls, intrusion detection/prevention systems, network monitoring sensors, web application firewalls, data loss prevention systems, and APM servers. The observer.* fields shall be populated with details of the system, if any, that detects, observes and/or creates a network, security, or application event or metric. Message queues and ETL components used in processing events or metrics are not considered observers in ECS. \n', - fields: { - 'observer.hostname': { - description: 'Hostname of the observer.', - example: '', - footnote: '', - group: 2, - level: 'core', - name: 'observer.hostname', - required: false, - type: 'keyword', - }, - 'observer.ip': { - description: 'IP address of the observer.', - example: '', - footnote: '', - group: 2, - level: 'core', - name: 'observer.ip', - required: false, - type: 'ip', - }, - 'observer.mac': { - description: 'MAC address of the observer', - example: '', - footnote: '', - group: 2, - level: 'core', - name: 'observer.mac', - required: false, - type: 'keyword', - }, - 'observer.serial_number': { - description: 'Observer serial number.', - example: '', - footnote: '', - group: 2, - level: 'extended', - name: 'observer.serial_number', - required: false, - type: 'keyword', - }, - 'observer.type': { - description: - 'The type of the observer the data is coming from.\nThere is no predefined list of observer types. Some examples are `forwarder`, `firewall`, `ids`, `ips`, `proxy`, `poller`, `sensor`, `APM server`.', - example: 'firewall', - footnote: '', - group: 2, - level: 'core', - name: 'observer.type', - required: false, - type: 'keyword', - }, - 'observer.vendor': { - description: 'observer vendor information.', - example: '', - footnote: '', - group: 2, - level: 'core', - name: 'observer.vendor', - required: false, - type: 'keyword', - }, - 'observer.version': { - description: 'Observer version.', - example: '', - footnote: '', - group: 2, - level: 'core', - name: 'observer.version', - required: false, - type: 'keyword', - }, - }, - group: 2, - name: 'observer', - title: 'Observer', - type: 'group', - }, - organization: { - description: - 'The organization fields enrich data with information about the company or entity the data is associated with. These fields help you arrange or filter data stored in an index by one or multiple organizations.\n', - fields: { - 'organization.id': { - description: 'Unique identifier for the organization.', - example: '', - footnote: '', - group: 2, - level: 'extended', - name: 'organization.id', - required: false, - type: 'keyword', - }, - 'organization.name': { - description: 'Organization name.', - example: '', - footnote: '', - group: 2, - level: 'extended', - name: 'organization.name', - required: false, - type: 'keyword', - }, - }, - group: 2, - name: 'organization', - title: 'Organization', - type: 'group', - }, - os: { - description: 'The OS fields contain information about the operating system.\n', - fields: { - 'os.family': { - description: 'OS family (such as redhat, debian, freebsd, windows).', - example: 'debian', - footnote: '', - group: 2, - level: 'extended', - name: 'os.family', - required: false, - type: 'keyword', - }, - 'os.kernel': { - description: 'Operating system kernel version as a raw string.', - example: '4.4.0-112-generic', - footnote: '', - group: 2, - level: 'extended', - name: 'os.kernel', - required: false, - type: 'keyword', - }, - 'os.name': { - description: 'Operating system name.', - example: 'Mac OS X', - footnote: '', - group: 2, - level: 'extended', - name: 'os.name', - required: false, - type: 'keyword', - }, - 'os.platform': { - description: 'Operating system platform (such centos, ubuntu, windows).', - example: 'darwin', - footnote: '', - group: 2, - level: 'extended', - name: 'os.platform', - required: false, - type: 'keyword', - }, - 'os.version': { - description: 'Operating system version as a raw string.', - example: '10.12.6-rc2', - footnote: '', - group: 2, - level: 'extended', - name: 'os.version', - required: false, - type: 'keyword', - }, - }, - group: 2, - name: 'os', - title: 'Operating System', - type: 'group', - }, - process: { - description: - 'These fields contain information about a process. These fields can help you correlate metrics information with a process id/name from a log message. The `process.pid` often stays in the metric itself and is copied to the global field for correlation.\n', - fields: { - 'process.args': { - description: 'Process arguments.\nMay be filtered to protect sensitive information.', - example: "['ssh', '-l', 'user', '10.0.0.16']", - footnote: '', - group: 2, - level: 'extended', - name: 'process.args', - required: false, - type: 'keyword', - }, - 'process.executable': { - description: 'Absolute path to the process executable.', - example: '/usr/bin/ssh', - footnote: '', - group: 2, - level: 'extended', - name: 'process.executable', - required: false, - type: 'keyword', - }, - 'process.name': { - description: 'Process name.\nSometimes called program name or similar.', - example: 'ssh', - footnote: '', - group: 2, - level: 'extended', - name: 'process.name', - required: false, - type: 'keyword', - }, - 'process.pid': { - description: 'Process id.', - example: '', - footnote: '', - group: 2, - level: 'core', - name: 'process.pid', - required: false, - type: 'long', - }, - 'process.ppid': { - description: 'Process parent id.', - example: '', - footnote: '', - group: 2, - level: 'extended', - name: 'process.ppid', - required: false, - type: 'long', - }, - 'process.start': { - description: 'The time the process started.', - example: '2016-05-23T08:05:34.853Z', - footnote: '', - group: 2, - level: 'extended', - name: 'process.start', - required: false, - type: 'date', - }, - 'process.thread.id': { - description: 'Thread ID.', - example: '4242', - footnote: '', - group: 2, - level: 'extended', - name: 'process.thread.id', - required: false, - type: 'long', - }, - 'process.title': { description: - 'Process title.\nThe proctitle, some times the same as process name. Can also be different: for example a browser setting its title to the web page currently opened.', - example: '', - footnote: '', - group: 2, - level: 'extended', - name: 'process.title', - required: false, - type: 'keyword', - }, - 'process.working_directory': { - description: 'The working directory of the process.', - example: '/home/alice', - footnote: '', - group: 2, - level: 'extended', - name: 'process.working_directory', - required: false, - type: 'keyword', - }, - }, - group: 2, - name: 'process', - title: 'Process', - type: 'group', - }, - related: { - description: - 'This field set is meant to facilitate pivoting around a piece of data. Some pieces of information can be seen in many places in ECS. To facilitate searching for them, append values to their corresponding field in `related.`. A concrete example is IP addresses, which can be under host, observer, source, destination, client, server, and network.forwarded_ip. If you append all IPs to `related.ip`, you can then search for a given IP trivially, no matter where it appeared, by querying `related.ip:a.b.c.d`.\n', - fields: { - 'related.ip': { - description: 'All of the IPs seen on your event.', - example: '', - footnote: '', - group: 2, - level: 'extended', - name: 'related.ip', - required: false, - type: 'ip', - }, - }, - group: 2, - name: 'related', - title: 'Related', - type: 'group', - }, - server: { - description: - 'A Server is defined as the responder in a network connection for events regarding sessions, connections, or bidirectional flow records. For TCP events, the server is the receiver of the initial SYN packet(s) of the TCP connection. For other protocols, the server is generally the responder in the network transaction. Some systems actually use the term "responder" to refer the server in TCP connections. The server fields describe details about the system acting as the server in the network event. Server fields are usually populated in conjunction with client fields. Server fields are generally not populated for packet-level events.\n', - fields: { - 'server.bytes': { - description: 'Bytes sent from the server to the client.', - example: '184', - footnote: '', - group: 2, - level: 'core', - name: 'server.bytes', - required: false, - type: 'long', - }, - 'server.domain': { - description: 'Server domain.', - example: '', - footnote: '', - group: 2, - level: 'core', - name: 'server.domain', - required: false, - type: 'keyword', - }, - 'server.ip': { - description: 'IP address of the server.\nCan be one or multiple IPv4 or IPv6 addresses.', - example: '', - footnote: '', - group: 2, - level: 'core', - name: 'server.ip', - required: false, - type: 'ip', - }, - 'server.mac': { - description: 'MAC address of the server.', - example: '', - footnote: '', - group: 2, - level: 'core', - name: 'server.mac', - required: false, - type: 'keyword', - }, - 'server.packets': { - description: 'Packets sent from the server to the client.', - example: '12', - footnote: '', - group: 2, - level: 'core', - name: 'server.packets', - required: false, - type: 'long', - }, - 'server.port': { - description: 'Port of the server.', - example: '', - footnote: '', - group: 2, - level: 'core', - name: 'server.port', - required: false, - type: 'long', - }, - }, - group: 2, - name: 'server', - title: 'Server', - type: 'group', - }, - service: { - description: - 'The service fields describe the service for or from which the data was collected. These fields help you find and correlate logs for a specific service and version.\n', - fields: { - 'service.ephemeral_id': { - description: - 'Ephemeral identifier of this service (if one exists).\nThis id normally changes across restarts, but `service.id` does not.', - example: '8a4f500f', - footnote: '', - group: 2, - level: 'extended', - name: 'service.ephemeral_id', - required: false, - type: 'keyword', - }, - 'service.id': { - description: - 'Unique identifier of the running service.\nThis id should uniquely identify this service. This makes it possible to correlate logs and metrics for one specific service.\nExample: If you are experiencing issues with one redis instance, you can filter on that id to see metrics and logs for that single instance.', - example: 'd37e5ebfe0ae6c4972dbe9f0174a1637bb8247f6', - footnote: '', - group: 2, - level: 'core', - name: 'service.id', - required: false, - type: 'keyword', - }, - 'service.name': { - description: - 'Name of the service data is collected from.\nThe name of the service is normally user given. This allows if two instances of the same service are running on the same machine they can be differentiated by the `service.name`.\nAlso it allows for distributed services that run on multiple hosts to correlate the related instances based on the name.\nIn the case of Elasticsearch the service.name could contain the cluster name. For Beats the service.name is by default a copy of the `service.type` field if no name is specified.', - example: 'elasticsearch-metrics', - footnote: '', - group: 2, - level: 'core', - name: 'service.name', - required: false, - type: 'keyword', - }, - 'service.state': { - description: 'Current state of the service.', - example: '', - footnote: '', - group: 2, - level: 'core', - name: 'service.state', - required: false, - type: 'keyword', - }, - 'service.type': { - description: - 'The type of the service data is collected from.\nThe type can be used to group and correlate logs and metrics from one service type.\nExample: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`.', - example: 'elasticsearch', - footnote: '', - group: 2, - level: 'core', - name: 'service.type', - required: false, - type: 'keyword', - }, - 'service.version': { - description: - 'Version of the service the data was collected from.\nThis allows to look at a data set only for a specific version of a service.', - example: '3.2.4', - footnote: '', - group: 2, - level: 'core', - name: 'service.version', - required: false, - type: 'keyword', - }, - }, - group: 2, - name: 'service', - title: 'Service', - type: 'group', - }, - source: { - description: - 'Source fields describe details about the source of a packet/event. Source fields are usually populated in conjunction with destination fields.\n', - fields: { - 'source.bytes': { - description: 'Bytes sent from the source to the destination.', - example: '184', - footnote: '', - group: 2, - level: 'core', - name: 'source.bytes', - required: false, - type: 'long', - }, - 'source.domain': { - description: 'Source domain.', - example: '', - footnote: '', - group: 2, - level: 'core', - name: 'source.domain', - required: false, - type: 'keyword', - }, - 'source.ip': { - description: 'IP address of the source.\nCan be one or multiple IPv4 or IPv6 addresses.', - example: '', - footnote: '', - group: 2, - level: 'core', - name: 'source.ip', - required: false, - type: 'ip', - }, - 'source.mac': { - description: 'MAC address of the source.', - example: '', - footnote: '', - group: 2, - level: 'core', - name: 'source.mac', - required: false, - type: 'keyword', - }, - 'source.packets': { - description: 'Packets sent from the source to the destination.', - example: '12', - footnote: '', - group: 2, - level: 'core', - name: 'source.packets', - required: false, - type: 'long', - }, - 'source.port': { - description: 'Port of the source.', - example: '', - footnote: '', - group: 2, - level: 'core', - name: 'source.port', - required: false, - type: 'long', - }, - }, - group: 2, - name: 'source', - title: 'Source', - type: 'group', - }, - url: { - description: 'URL fields provide a complete URL, with scheme, host, and path.\n', - fields: { - 'url.domain': { - description: - 'Domain of the request, such as "www.elastic.co".\nIn some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field.', - example: 'www.elastic.co', - footnote: '', - group: 2, - level: 'extended', - name: 'url.domain', - required: false, - type: 'keyword', - }, - 'url.fragment': { - description: - 'Portion of the url after the `#`, such as "top".\nThe `#` is not part of the fragment.', - example: '', - footnote: '', - group: 2, - level: 'extended', - name: 'url.fragment', - required: false, - type: 'keyword', - }, - 'url.full': { - description: - 'If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source.', - example: 'https://www.elastic.co:443/search?q=elasticsearch#top', - footnote: '', - group: 2, - level: 'extended', - name: 'url.full', - required: false, - type: 'keyword', - }, - 'url.original': { - description: - 'Unmodified original url as seen in the event source.\nNote that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path.\nThis field is meant to represent the URL as it was observed, complete or not.', - example: 'https://www.elastic.co:443/search?q=elasticsearch#top or /search?q=elasticsearch', - footnote: '', - group: 2, - level: 'extended', - name: 'url.original', - required: false, - type: 'keyword', - }, - 'url.password': { - description: 'Password of the request.', - example: '', - footnote: '', - group: 2, - level: 'extended', - name: 'url.password', - required: false, - type: 'keyword', - }, - 'url.path': { - description: 'Path of the request, such as "/search".', - example: '', - footnote: '', - group: 2, - level: 'extended', - name: 'url.path', - required: false, - type: 'keyword', - }, - 'url.port': { - description: 'Port of the request, such as 443.', - example: '443', - footnote: '', - group: 2, - level: 'extended', - name: 'url.port', - required: false, - type: 'integer', - }, - 'url.query': { - description: - 'The query field describes the query string of the request, such as "q=elasticsearch".\nThe `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases.', - example: '', - footnote: '', - group: 2, - level: 'extended', - name: 'url.query', - required: false, - type: 'keyword', - }, - 'url.scheme': { - description: - 'Scheme of the request, such as "https".\nNote: The `:` is not part of the scheme.', - example: 'https', - footnote: '', - group: 2, - level: 'extended', - name: 'url.scheme', - required: false, - type: 'keyword', - }, - 'url.username': { - description: 'Username of the request.', - example: '', - footnote: '', - group: 2, - level: 'extended', - name: 'url.username', - required: false, - type: 'keyword', - }, - }, - group: 2, - name: 'url', - title: 'URL', - type: 'group', - }, - user: { - description: - 'The user fields describe information about the user that is relevant to the event. Fields can have one entry or multiple entries. If a user has more than one id, provide an array that includes all of them.\n', - fields: { - 'user.email': { - description: 'User email address.', - example: '', - footnote: '', - group: 2, - level: 'extended', - name: 'user.email', - required: false, - type: 'keyword', - }, - 'user.full_name': { - description: "User's full name, if available.", - example: 'Albert Einstein', - footnote: '', - group: 2, - level: 'extended', - name: 'user.full_name', - required: false, - type: 'keyword', - }, - 'user.group': { - description: - 'Group the user is a part of. This field can contain a list of groups, if necessary.', - example: '', - footnote: '', - group: 2, - level: 'extended', - name: 'user.group', - required: false, - type: 'keyword', - }, - 'user.hash': { - description: - 'Unique user hash to correlate information for a user in anonymized form.\nUseful if `user.id` or `user.name` contain confidential information and cannot be used.', - example: '', - footnote: '', - group: 2, - level: 'extended', - name: 'user.hash', - required: false, - type: 'keyword', - }, - 'user.id': { - description: 'One or multiple unique identifiers of the user.', - example: '', - footnote: '', - group: 2, - level: 'core', - name: 'user.id', - required: false, - type: 'keyword', - }, - 'user.name': { - description: 'Short name or login of the user.', - example: 'albert', - footnote: '', - group: 2, - level: 'core', - name: 'user.name', - required: false, - type: 'keyword', - }, - }, - group: 2, - name: 'user', - title: 'User', - type: 'group', - }, - user_agent: { - description: - 'The user_agent fields normally come from a browser request. They often show up in web service logs coming from the parsed user agent string.\n', - fields: { - 'user_agent.device.name': { - description: 'Name of the device.', - example: 'iPhone', - footnote: '', - group: 2, - level: 'extended', - name: 'user_agent.device.name', - required: false, - type: 'keyword', - }, - 'user_agent.name': { - description: 'Name of the user agent.', - example: 'Safari', - footnote: '', - group: 2, - level: 'extended', - name: 'user_agent.name', - required: false, - type: 'keyword', - }, - 'user_agent.original': { - description: 'Unparsed version of the user_agent.', - example: - 'Mozilla/5.0 (iPhone; CPU iPhone OS 12_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1', - footnote: '', - group: 2, - level: 'extended', - name: 'user_agent.original', - required: false, - type: '(not indexed)', - }, - 'user_agent.version': { - description: 'Version of the user agent.', - example: '12.0', - footnote: '', - group: 2, - level: 'extended', - name: 'user_agent.version', - required: false, - type: 'keyword', - }, - }, - group: 2, - name: 'user_agent', - title: 'User agent', - type: 'group', + 'The vulnerability fields describe information about a vulnerability\nthat is relevant to an event.', + type: 'group', + fields: [ + { + name: 'category', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The type of system or architecture that the vulnerability affects.\nThese may be platform-specific (for example, Debian or SUSE) or general (for\nexample, Database or Firewall). For example (https://qualysguard.qualys.com/qwebhelp/fo_portal/knowledgebase/vulnerability_categories.htm[Qualys\nvulnerability categories])\n\nThis field must be an array.', + example: '["Firewall"]', + default_field: false, + }, + { + name: 'classification', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The classification of the vulnerability scoring system. For example\n(https://www.first.org/cvss/)', + example: 'CVSS', + default_field: false, + }, + { + name: 'description', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + }, + ], + description: + 'The description of the vulnerability that provides additional context\nof the vulnerability. For example (https://cve.mitre.org/about/faqs.html#cve_entry_descriptions_created[Common\nVulnerabilities and Exposure CVE description])', + example: 'In macOS before 2.12.6, there is a vulnerability in the RPC...', + default_field: false, + }, + { + name: 'enumeration', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The type of identifier used for this vulnerability. For example\n(https://cve.mitre.org/about/)', + example: 'CVE', + default_field: false, + }, + { + name: 'id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The identification (ID) is the number portion of a vulnerability\nentry. It includes a unique identification number for the vulnerability. For\nexample (https://cve.mitre.org/about/faqs.html#what_is_cve_id)[Common Vulnerabilities\nand Exposure CVE ID]', + example: 'CVE-2019-00001', + default_field: false, + }, + { + name: 'reference', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'A resource that provides additional information, context, and mitigations\nfor the identified vulnerability.', + example: 'https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-6111', + default_field: false, + }, + { + name: 'report_id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'The report or scan identification number.', + example: 20191018.0001, + default_field: false, + }, + { + name: 'scanner.vendor', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'The name of the vulnerability scanner vendor.', + example: 'Tenable', + default_field: false, + }, + { + name: 'score.base', + level: 'extended', + type: 'float', + description: + 'Scores can range from 0.0 to 10.0, with 10.0 being the most severe.\n\nBase scores cover an assessment for exploitability metrics (attack vector,\ncomplexity, privileges, and user interaction), impact metrics (confidentiality,\nintegrity, and availability), and scope. For example (https://www.first.org/cvss/specification-document)', + example: 5.5, + default_field: false, + }, + { + name: 'score.environmental', + level: 'extended', + type: 'float', + description: + 'Scores can range from 0.0 to 10.0, with 10.0 being the most severe.\n\nEnvironmental scores cover an assessment for any modified Base metrics, confidentiality,\nintegrity, and availability requirements. For example (https://www.first.org/cvss/specification-document)', + example: 5.5, + default_field: false, + }, + { + name: 'score.temporal', + level: 'extended', + type: 'float', + description: + 'Scores can range from 0.0 to 10.0, with 10.0 being the most severe.\n\nTemporal scores cover an assessment for code maturity, remediation level,\nand confidence. For example (https://www.first.org/cvss/specification-document)', + default_field: false, + }, + { + name: 'score.version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The National Vulnerability Database (NVD) provides qualitative\nseverity rankings of "Low", "Medium", and "High" for CVSS v2.0 base score\nranges in addition to the severity ratings for CVSS v3.0 as they are defined\nin the CVSS v3.0 specification.\n\nCVSS is owned and managed by FIRST.Org, Inc. (FIRST), a US-based non-profit\norganization, whose mission is to help computer security incident response\nteams across the world. For example (https://nvd.nist.gov/vuln-metrics/cvss)', + example: 2, + default_field: false, + }, + { + name: 'severity', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The severity of the vulnerability can help with metrics and internal\nprioritization regarding remediation. For example (https://nvd.nist.gov/vuln-metrics/cvss)', + example: 'Critical', + default_field: false, + }, + ], + }, + ], }, -}; +]; diff --git a/x-pack/legacy/plugins/siem/server/utils/beat_schema/8.0.0/filebeat.ts b/x-pack/legacy/plugins/siem/server/utils/beat_schema/8.0.0/filebeat.ts index a5877f6c34b8f..3b8c92ebba269 100644 --- a/x-pack/legacy/plugins/siem/server/utils/beat_schema/8.0.0/filebeat.ts +++ b/x-pack/legacy/plugins/siem/server/utils/beat_schema/8.0.0/filebeat.ts @@ -15,91 +15,129 @@ export const filebeatSchema: Schema = [ { key: 'ecs', title: 'ECS', - description: 'ECS fields.', + description: 'ECS Fields.', fields: [ { name: '@timestamp', - type: 'date', level: 'core', required: true, - example: '2016-05-23T08:05:34.853Z', + type: 'date', description: - 'Date/time when the event originated. For log events this is the date/time when the event was generated, and not when it was read. Required field for all events.', - }, - { - name: 'tags', - level: 'core', - type: 'keyword', - example: '["production", "env2"]', - description: 'List of keywords used to tag each event.', + 'Date/time when the event originated.\n\nThis is the date/time extracted from the event, typically representing when\nthe event was generated by the source.\n\nIf the event source has no original timestamp, this value is typically populated\nby the first time the event was received by the pipeline.\n\nRequired field for all events.', + example: '2016-05-23T08:05:34.853Z', }, { name: 'labels', level: 'core', type: 'object', - example: { - env: 'production', - application: 'foo-bar', - }, + object_type: 'keyword', description: - 'Key/value pairs. Can be used to add meta information to events. Should not contain nested objects. All values are stored as keyword. Example: `docker` and `k8s` labels.', + 'Custom key/value pairs.\n\nCan be used to add meta information to events. Should not contain nested objects.\nAll values are stored as keyword.\n\nExample: `docker` and `k8s` labels.', + example: '{"application": "foo-bar", "env": "production"}', }, { name: 'message', level: 'core', type: 'text', - example: 'Hello World', description: - 'For log events the message field contains the log message. In other use cases the message field can be used to concatenate different values which are then freely searchable. If multiple messages exist, they can be combined into one message.', + 'For log events the message field contains the log message, optimized\nfor viewing in a log viewer.\n\nFor structured logs without an original message field, other fields can be concatenated\nto form a human-readable summary of the event.\n\nIf multiple messages exist, they can be combined into one message.', + example: 'Hello World', + }, + { + name: 'tags', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'List of keywords used to tag each event.', + example: '["production", "env2"]', }, { name: 'agent', title: 'Agent', group: 2, description: - 'The agent fields contain the data about the software entity, if any, that collects, detects, or observes events on a host, or takes measurements on a host. Examples include Beats. Agents may also run on observers. ECS agent.* fields shall be populated with details of the agent running on the host or observer where the event happened or the measurement was taken.', + 'The agent fields contain the data about the software entity, if\nany, that collects, detects, or observes events on a host, or takes measurements\non a host.\n\nExamples include Beats. Agents may also run on observers. ECS agent.* fields\nshall be populated with details of the agent running on the host or observer\nwhere the event happened or the measurement was taken.', footnote: - 'Examples: In the case of Beats for logs, the agent.name is filebeat. For APM, it is the agent running in the app/service. The agent information does not change if data is sent through queuing systems like Kafka, Redis, or processing systems such as Logstash or APM Server.', + 'Examples: In the case of Beats for logs, the agent.name is filebeat.\nFor APM, it is the agent running in the app/service. The agent information does\nnot change if data is sent through queuing systems like Kafka, Redis, or processing\nsystems such as Logstash or APM Server.', type: 'group', fields: [ { - name: 'version', + name: 'ephemeral_id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Ephemeral identifier of this agent (if one exists).\n\nThis id normally changes across restarts, but `agent.id` does not.', + example: '8a4f500f', + }, + { + name: 'id', level: 'core', type: 'keyword', - description: 'Version of the agent.', - example: '6.0.0-rc2', + ignore_above: 1024, + description: + 'Unique identifier of this agent (if one exists).\n\nExample: For Beats this would be beat.id.', + example: '8a4f500d', }, { name: 'name', level: 'core', type: 'keyword', + ignore_above: 1024, description: - 'Name of the agent. This is a name that can be given to an agent. This can be helpful if for example two Filebeat instances are running on the same host but a human readable separation is needed on which Filebeat instance data is coming from. If no name is given, the name is often left empty.', + 'Custom name of the agent.\n\nThis is a name that can be given to an agent. This can be helpful if for example\ntwo Filebeat instances are running on the same host but a human readable separation\nis needed on which Filebeat instance data is coming from.\n\nIf no name is given, the name is often left empty.', example: 'foo', }, { name: 'type', level: 'core', type: 'keyword', + ignore_above: 1024, description: - 'Type of the agent. The agent type stays always the same and should be given by the agent used. In case of Filebeat the agent would always be Filebeat also if two Filebeat instances are run on the same machine.', + 'Type of the agent.\n\nThe agent type stays always the same and should be given by the agent used.\nIn case of Filebeat the agent would always be Filebeat also if two Filebeat\ninstances are run on the same machine.', example: 'filebeat', }, { - name: 'id', + name: 'version', level: 'core', type: 'keyword', + ignore_above: 1024, + description: 'Version of the agent.', + example: '6.0.0-rc2', + }, + ], + }, + { + name: 'as', + title: 'Autonomous System', + group: 2, + description: + 'An autonomous system (AS) is a collection of connected Internet Protocol\n(IP) routing prefixes under the control of one or more network operators on\nbehalf of a single administrative entity or domain that presents a common, clearly\ndefined routing policy to the internet.', + type: 'group', + fields: [ + { + name: 'number', + level: 'extended', + type: 'long', description: - 'Unique identifier of this agent (if one exists). Example: For Beats this would be beat.id.', - example: '8a4f500d', + 'Unique number allocated to the autonomous system. The autonomous\nsystem number (ASN) uniquely identifies each network on the Internet.', + example: 15169, }, { - name: 'ephemeral_id', + name: 'organization.name', level: 'extended', type: 'keyword', - description: - 'Ephemeral identifier of this agent (if one exists). This id normally changes across restarts, but `agent.id` does not.', - example: '8a4f500f', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Organization name.', + example: 'Google LLC', }, ], }, @@ -108,7183 +146,18212 @@ export const filebeatSchema: Schema = [ title: 'Client', group: 2, description: - 'A client is defined as the initiator of a network connection for events regarding sessions, connections, or bidirectional flow records. For TCP events, the client is the initiator of the TCP connection that sends the SYN packet(s). For other protocols, the client is generally the initiator or requestor in the network transaction. Some systems use the term "originator" to refer the client in TCP connections. The client fields describe details about the system acting as the client in the network event. Client fields are usually populated in conjunction with server fields. Client fields are generally not populated for packet-level events. Client / server representations can add semantic context to an exchange, which is helpful to visualize the data in certain situations. If your context falls in that category, you should still ensure that source and destination are filled appropriately.', + 'A client is defined as the initiator of a network connection for\nevents regarding sessions, connections, or bidirectional flow records.\n\nFor TCP events, the client is the initiator of the TCP connection that sends\nthe SYN packet(s). For other protocols, the client is generally the initiator\nor requestor in the network transaction. Some systems use the term "originator"\nto refer the client in TCP connections. The client fields describe details about\nthe system acting as the client in the network event. Client fields are usually\npopulated in conjunction with server fields. Client fields are generally not\npopulated for packet-level events.\n\nClient / server representations can add semantic context to an exchange, which\nis helpful to visualize the data in certain situations. If your context falls\nin that category, you should still ensure that source and destination are filled\nappropriately.', type: 'group', fields: [ { name: 'address', level: 'extended', type: 'keyword', + ignore_above: 1024, description: - 'Some event client addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is.', + 'Some event client addresses are defined ambiguously. The event\nwill sometimes list an IP, a domain or a unix socket. You should always store\nthe raw address in the `.address` field.\n\nThen it should be duplicated to `.ip` or `.domain`, depending on which one\nit is.', }, { - name: 'ip', - level: 'core', - type: 'ip', - description: 'IP address of the client. Can be one or multiple IPv4 or IPv6 addresses.', + name: 'as.number', + level: 'extended', + type: 'long', + description: + 'Unique number allocated to the autonomous system. The autonomous\nsystem number (ASN) uniquely identifies each network on the Internet.', + example: 15169, }, { - name: 'port', + name: 'as.organization.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Organization name.', + example: 'Google LLC', + }, + { + name: 'bytes', level: 'core', type: 'long', - description: 'Port of the client.', + format: 'bytes', + description: 'Bytes sent from the client to the server.', + example: 184, }, { - name: 'mac', + name: 'domain', level: 'core', type: 'keyword', - description: 'MAC address of the client.', + ignore_above: 1024, + description: 'Client domain.', }, { - name: 'domain', + name: 'geo.city_name', level: 'core', type: 'keyword', - description: 'Client domain.', + ignore_above: 1024, + description: 'City name.', + example: 'Montreal', }, { - name: 'bytes', + name: 'geo.continent_name', level: 'core', - type: 'long', - format: 'bytes', - example: 184, - description: 'Bytes sent from the client to the server.', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the continent.', + example: 'North America', }, { - name: 'packets', + name: 'geo.country_iso_code', level: 'core', - type: 'long', - example: 12, - description: 'Packets sent from the client to the server.', + type: 'keyword', + ignore_above: 1024, + description: 'Country ISO code.', + example: 'CA', }, { - name: 'geo', - title: 'Geo', - group: 2, - description: - 'Geo fields can carry data about a specific location related to an event or geo information derived from an IP field.', - type: 'group', - fields: [ - { - name: 'location', - level: 'core', - type: 'geo_point', - description: 'Longitude and latitude.', - example: '{ "lon": -73.614830, "lat": 45.505918 }', - }, - { - name: 'continent_name', - level: 'core', - type: 'keyword', - description: 'Name of the continent.', - example: 'North America', - }, - { - name: 'country_name', - level: 'core', - type: 'keyword', - description: 'Country name.', - example: 'Canada', - }, - { - name: 'region_name', - level: 'core', - type: 'keyword', - description: 'Region name.', - example: 'Quebec', - }, - { - name: 'city_name', - level: 'core', - type: 'keyword', - description: 'City name.', - example: 'Montreal', - }, - { - name: 'country_iso_code', - level: 'core', - type: 'keyword', - description: 'Country ISO code.', - example: 'CA', - }, - { - name: 'region_iso_code', - level: 'core', - type: 'keyword', - description: 'Region ISO code.', - example: 'CA-QC', - }, - { - name: 'name', - level: 'extended', - type: 'keyword', - description: - 'User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation.', - example: 'boston-dc', - }, - ], + name: 'geo.country_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Country name.', + example: 'Canada', }, - ], - }, - { - name: 'cloud', - title: 'Cloud', - group: 2, - description: 'Fields related to the cloud or infrastructure the events are coming from.', - footnote: - 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.', - type: 'group', - fields: [ { - name: 'provider', + name: 'geo.location', + level: 'core', + type: 'geo_point', + description: 'Longitude and latitude.', + example: '{ "lon": -73.614830, "lat": 45.505918 }', + }, + { + name: 'geo.name', level: 'extended', - example: 'ec2', type: 'keyword', + ignore_above: 1024, description: - 'Name of the cloud provider. Example values are ec2, gce, or digitalocean.', + 'User-defined description of a location, at the level of granularity\nthey care about.\n\nCould be the name of their data centers, the floor number, if this describes\na local physical entity, city names.\n\nNot typically used in automated geolocation.', + example: 'boston-dc', }, { - name: 'availability_zone', - level: 'extended', - example: 'us-east-1c', + name: 'geo.region_iso_code', + level: 'core', type: 'keyword', - description: 'Availability zone in which this host is running.', + ignore_above: 1024, + description: 'Region ISO code.', + example: 'CA-QC', }, { - name: 'region', - level: 'extended', + name: 'geo.region_name', + level: 'core', type: 'keyword', - example: 'us-east-1', - description: 'Region in which this host is running.', + ignore_above: 1024, + description: 'Region name.', + example: 'Quebec', }, { - name: 'instance.id', - level: 'extended', + name: 'ip', + level: 'core', + type: 'ip', + description: + 'IP address of the client.\n\nCan be one or multiple IPv4 or IPv6 addresses.', + }, + { + name: 'mac', + level: 'core', type: 'keyword', - example: 'i-1234567890abcdef0', - description: 'Instance ID of the host machine.', + ignore_above: 1024, + description: 'MAC address of the client.', }, { - name: 'instance.name', + name: 'nat.ip', level: 'extended', - type: 'keyword', - description: 'Instance name of the host machine.', + type: 'ip', + description: + 'Translated IP of source based NAT sessions (e.g. internal client\nto internet).\n\nTypically connections traversing load balancers, firewalls, or routers.', }, { - name: 'machine.type', + name: 'nat.port', + level: 'extended', + type: 'long', + format: 'string', + description: + 'Translated port of source based NAT sessions (e.g. internal client\nto internet).\n\nTypically connections traversing load balancers, firewalls, or routers.', + }, + { + name: 'packets', + level: 'core', + type: 'long', + description: 'Packets sent from the client to the server.', + example: 12, + }, + { + name: 'port', + level: 'core', + type: 'long', + format: 'string', + description: 'Port of the client.', + }, + { + name: 'registered_domain', level: 'extended', type: 'keyword', - example: 't2.medium', - description: 'Machine type of the host machine.', + ignore_above: 1024, + description: + 'The highest registered client domain, stripped of the subdomain.\n\nFor example, the registered domain for "foo.google.com" is "google.com".\n\nThis value can be determined precisely with a list like the public suffix\nlist (http://publicsuffix.org). Trying to approximate this by simply taking\nthe last two labels will not work well for TLDs such as "co.uk".', + example: 'google.com', }, { - name: 'account.id', + name: 'top_level_domain', level: 'extended', type: 'keyword', - example: 666777888999, + ignore_above: 1024, description: - 'The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.', + 'The effective top level domain (eTLD), also known as the domain\nsuffix, is the last part of the domain name. For example, the top level domain\nfor google.com is "com".\n\nThis value can be determined precisely with a list like the public suffix\nlist (http://publicsuffix.org). Trying to approximate this by simply taking\nthe last label will not work well for effective TLDs such as "co.uk".', + example: 'co.uk', }, - ], - }, - { - name: 'container', - title: 'Container', - group: 2, - description: - 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.', - type: 'group', - fields: [ { - name: 'runtime', + name: 'user.domain', level: 'extended', type: 'keyword', - description: 'Runtime managing this container.', - example: 'docker', + ignore_above: 1024, + description: + 'Name of the directory the user is a member of.\n\nFor example, an LDAP or Active Directory domain name.', }, { - name: 'id', - level: 'core', + name: 'user.email', + level: 'extended', type: 'keyword', - description: 'Unique container id.', + ignore_above: 1024, + description: 'User email address.', }, { - name: 'image.name', + name: 'user.full_name', level: 'extended', type: 'keyword', - description: 'Name of the image the container was built on.', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: "User's full name, if available.", + example: 'Albert Einstein', }, { - name: 'image.tag', + name: 'user.group.domain', level: 'extended', type: 'keyword', - description: 'Container image tag.', + ignore_above: 1024, + description: + 'Name of the directory the group is a member of.\n\nFor example, an LDAP or Active Directory domain name.', }, { - name: 'name', + name: 'user.group.id', level: 'extended', type: 'keyword', - description: 'Container name.', + ignore_above: 1024, + description: 'Unique identifier for the group on the system/platform.', }, { - name: 'labels', + name: 'user.group.name', level: 'extended', - type: 'object', - object_type: 'keyword', - description: 'Image labels.', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the group.', }, - ], - }, - { - name: 'destination', - title: 'Destination', - group: 2, - description: - 'Destination fields describe details about the destination of a packet/event. Destination fields are usually populated in conjunction with source fields.', - type: 'group', - fields: [ { - name: 'address', + name: 'user.hash', level: 'extended', type: 'keyword', + ignore_above: 1024, description: - 'Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is.', + 'Unique user hash to correlate information for a user in anonymized\nform.\n\nUseful if `user.id` or `user.name` contain confidential information and cannot\nbe used.', }, { - name: 'ip', + name: 'user.id', level: 'core', - type: 'ip', - description: - 'IP address of the destination. Can be one or multiple IPv4 or IPv6 addresses.', + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifiers of the user.', }, { - name: 'port', + name: 'user.name', level: 'core', - type: 'long', - description: 'Port of the destination.', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Short name or login of the user.', + example: 'albert', }, + ], + }, + { + name: 'cloud', + title: 'Cloud', + group: 2, + description: 'Fields related to the cloud or infrastructure the events are coming\nfrom.', + footnote: + 'Examples: If Metricbeat is running on an EC2 host and fetches data\nfrom its host, the cloud info contains the data about this machine. If Metricbeat\nruns on a remote machine outside the cloud and fetches data from a service running\nin the cloud, the field contains cloud data from the machine the service is\nrunning on.', + type: 'group', + fields: [ { - name: 'mac', - level: 'core', + name: 'account.id', + level: 'extended', type: 'keyword', - description: 'MAC address of the destination.', + ignore_above: 1024, + description: + 'The cloud account or organization id used to identify different\nentities in a multi-tenant environment.\n\nExamples: AWS account id, Google Cloud ORG Id, or other unique identifier.', + example: 666777888999, }, { - name: 'domain', - level: 'core', + name: 'availability_zone', + level: 'extended', type: 'keyword', - description: 'Destination domain.', + ignore_above: 1024, + description: 'Availability zone in which this host is running.', + example: 'us-east-1c', }, { - name: 'bytes', - level: 'core', - type: 'long', - format: 'bytes', - example: 184, - description: 'Bytes sent from the destination to the source.', + name: 'instance.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Instance ID of the host machine.', + example: 'i-1234567890abcdef0', }, { - name: 'packets', - level: 'core', - type: 'long', - example: 12, - description: 'Packets sent from the destination to the source.', + name: 'instance.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Instance name of the host machine.', + }, + { + name: 'machine.type', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Machine type of the host machine.', + example: 't2.medium', }, { - name: 'geo', - title: 'Geo', - group: 2, + name: 'provider', + level: 'extended', + type: 'keyword', + ignore_above: 1024, description: - 'Geo fields can carry data about a specific location related to an event or geo information derived from an IP field.', - type: 'group', - fields: [ - { - name: 'location', - level: 'core', - type: 'geo_point', - description: 'Longitude and latitude.', - example: '{ "lon": -73.614830, "lat": 45.505918 }', - }, - { - name: 'continent_name', - level: 'core', - type: 'keyword', - description: 'Name of the continent.', - example: 'North America', - }, - { - name: 'country_name', - level: 'core', - type: 'keyword', - description: 'Country name.', - example: 'Canada', - }, - { - name: 'region_name', - level: 'core', - type: 'keyword', - description: 'Region name.', - example: 'Quebec', - }, - { - name: 'city_name', - level: 'core', - type: 'keyword', - description: 'City name.', - example: 'Montreal', - }, - { - name: 'country_iso_code', - level: 'core', - type: 'keyword', - description: 'Country ISO code.', - example: 'CA', - }, - { - name: 'region_iso_code', - level: 'core', - type: 'keyword', - description: 'Region ISO code.', - example: 'CA-QC', - }, - { - name: 'name', - level: 'extended', - type: 'keyword', - description: - 'User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation.', - example: 'boston-dc', - }, - ], + 'Name of the cloud provider. Example values are aws, azure, gcp,\nor digitalocean.', + example: 'aws', + }, + { + name: 'region', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Region in which this host is running.', + example: 'us-east-1', }, ], }, { - name: 'ecs', - title: 'ECS', + name: 'code_signature', + title: 'Code Signature', group: 2, - description: 'Meta-information specific to ECS.', + description: 'These fields contain information about binary code signatures.', type: 'group', fields: [ { - name: 'version', + name: 'exists', level: 'core', + type: 'boolean', + description: 'Boolean to capture if a signature is present.', + example: 'true', + default_field: false, + }, + { + name: 'status', + level: 'extended', type: 'keyword', - required: true, + ignore_above: 1024, + description: + 'Additional information about the certificate status.\n\nThis is useful for logging cryptographic errors with the certificate validity\nor trust status. Leave unpopulated if the validity or trust of the certificate\nwas unchecked.', + example: 'ERROR_UNTRUSTED_ROOT', + default_field: false, + }, + { + name: 'subject_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Subject name of the code signer', + example: 'Microsoft Corporation', + default_field: false, + }, + { + name: 'trusted', + level: 'extended', + type: 'boolean', + description: + 'Stores the trust status of the certificate chain.\n\nValidating the trust of the certificate chain may be complicated, and this\nfield should only be populated by tools that actively check the status.', + example: 'true', + default_field: false, + }, + { + name: 'valid', + level: 'extended', + type: 'boolean', description: - 'ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. The current version is 1.0.0-beta2 .', - example: '1.0.0-beta2', + 'Boolean to capture if the digital signature is verified against\nthe binary content.\n\nLeave unpopulated if a certificate was unchecked.', + example: 'true', + default_field: false, }, ], }, { - name: 'error', - title: 'Error', + name: 'container', + title: 'Container', group: 2, description: - 'These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error.', + 'Container fields are used for meta information about the specific\ncontainer that is the source of information.\n\nThese fields help correlate data based containers from any runtime.', type: 'group', fields: [ { name: 'id', level: 'core', type: 'keyword', - description: 'Unique identifier for the error.', + ignore_above: 1024, + description: 'Unique container id.', }, { - name: 'message', - level: 'core', - type: 'text', - description: 'Error message.', + name: 'image.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the image the container was built on.', }, { - name: 'code', - level: 'core', + name: 'image.tag', + level: 'extended', type: 'keyword', - description: 'Error code describing the error.', + ignore_above: 1024, + description: 'Container image tags.', + }, + { + name: 'labels', + level: 'extended', + type: 'object', + object_type: 'keyword', + description: 'Image labels.', + }, + { + name: 'name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Container name.', + }, + { + name: 'runtime', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Runtime managing this container.', + example: 'docker', }, ], }, { - name: 'event', - title: 'Event', + name: 'destination', + title: 'Destination', group: 2, description: - 'The event fields are used for context information about the log or metric event itself. A log is defined as an event containing details of something that happened. Log events must include the time at which the thing happened. Examples of log events include a process starting on a host, a network packet being sent from a source to a destination, or a network connection between a client and a server being initiated or closed. A metric is defined as an event containing one or more numerical or categorical measurements and the time at which the measurement was taken. Examples of metric events include memory pressure measured on a host, or vulnerabilities measured on a scanned host.', + 'Destination fields describe details about the destination of a packet/event.\n\nDestination fields are usually populated in conjunction with source fields.', type: 'group', fields: [ { - name: 'id', - level: 'core', + name: 'address', + level: 'extended', type: 'keyword', - description: 'Unique ID to describe the event.', - example: '8a4f500d', + ignore_above: 1024, + description: + 'Some event destination addresses are defined ambiguously. The\nevent will sometimes list an IP, a domain or a unix socket. You should always\nstore the raw address in the `.address` field.\n\nThen it should be duplicated to `.ip` or `.domain`, depending on which one\nit is.', }, { - name: 'kind', + name: 'as.number', level: 'extended', - type: 'keyword', + type: 'long', description: - 'The kind of the event. This gives information about what type of information the event contains, without being specific to the contents of the event. Examples are `event`, `state`, `alarm`. Warning: In future versions of ECS, we plan to provide a list of acceptable values for this field, please use with caution.', - example: 'state', + 'Unique number allocated to the autonomous system. The autonomous\nsystem number (ASN) uniquely identifies each network on the Internet.', + example: 15169, }, { - name: 'category', - level: 'core', + name: 'as.organization.name', + level: 'extended', type: 'keyword', - description: - 'Event category. This contains high-level information about the contents of the event. It is more generic than `event.action`, in the sense that typically a category contains multiple actions. Warning: In future versions of ECS, we plan to provide a list of acceptable values for this field, please use with caution.', - example: 'user-management', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Organization name.', + example: 'Google LLC', }, { - name: 'action', + name: 'bytes', level: 'core', - type: 'keyword', - description: - 'The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer.', - example: 'user-password-change', + type: 'long', + format: 'bytes', + description: 'Bytes sent from the destination to the source.', + example: 184, }, { - name: 'outcome', - level: 'extended', + name: 'domain', + level: 'core', type: 'keyword', - description: - 'The outcome of the event. If the event describes an action, this fields contains the outcome of that action. Examples outcomes are `success` and `failure`. Warning: In future versions of ECS, we plan to provide a list of acceptable values for this field, please use with caution.', - example: 'success', + ignore_above: 1024, + description: 'Destination domain.', }, { - name: 'type', + name: 'geo.city_name', level: 'core', type: 'keyword', - description: 'Reserved for future usage. Please avoid using this field for user data.', + ignore_above: 1024, + description: 'City name.', + example: 'Montreal', }, { - name: 'module', + name: 'geo.continent_name', level: 'core', type: 'keyword', - description: - 'Name of the module this data is coming from. This information is coming from the modules used in Beats or Logstash.', - example: 'mysql', + ignore_above: 1024, + description: 'Name of the continent.', + example: 'North America', }, { - name: 'dataset', + name: 'geo.country_iso_code', level: 'core', type: 'keyword', - description: - 'Name of the dataset. The concept of a `dataset` (fileset / metricset) is used in Beats as a subset of modules. It contains the information which is currently stored in metricset.name and metricset.module or fileset.name.', - example: 'stats', + ignore_above: 1024, + description: 'Country ISO code.', + example: 'CA', }, { - name: 'severity', + name: 'geo.country_name', level: 'core', - type: 'long', - example: '7', - description: - "Severity describes the severity of the event. What the different severity values mean can very different between use cases. It's up to the implementer to make sure severities are consistent across events. ", + type: 'keyword', + ignore_above: 1024, + description: 'Country name.', + example: 'Canada', }, { - name: 'original', + name: 'geo.location', level: 'core', - type: 'keyword', - example: - 'Sep 19 08:26:10 host CEF:0|Security| threatmanager|1.0|100| worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2spt=1232', - description: - 'Raw text message of entire event. Used to demonstrate log integrity. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`.', - index: false, - doc_values: false, + type: 'geo_point', + description: 'Longitude and latitude.', + example: '{ "lon": -73.614830, "lat": 45.505918 }', }, { - name: 'hash', + name: 'geo.name', level: 'extended', type: 'keyword', - example: '123456789012345678901234567890ABCD', + ignore_above: 1024, description: - 'Hash (perhaps logstash fingerprint) of raw field to be able to demonstrate log integrity.', + 'User-defined description of a location, at the level of granularity\nthey care about.\n\nCould be the name of their data centers, the floor number, if this describes\na local physical entity, city names.\n\nNot typically used in automated geolocation.', + example: 'boston-dc', }, { - name: 'duration', + name: 'geo.region_iso_code', level: 'core', - type: 'long', - format: 'duration', - input_format: 'nanoseconds', - description: - 'Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time.', + type: 'keyword', + ignore_above: 1024, + description: 'Region ISO code.', + example: 'CA-QC', }, { - name: 'timezone', - level: 'extended', + name: 'geo.region_name', + level: 'core', type: 'keyword', - description: - 'This field should be populated when the event\'s timestamp does not include timezone information already (e.g. default Syslog timestamps). It\'s optional otherwise. Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00").', + ignore_above: 1024, + description: 'Region name.', + example: 'Quebec', }, { - name: 'created', + name: 'ip', level: 'core', - type: 'date', + type: 'ip', description: - 'event.created contains the date when the event was created. This timestamp is distinct from @timestamp in that @timestamp contains the processed timestamp. For logs these two timestamps can be different as the timestamp in the log line and when the event is read for example by Filebeat are not identical. `@timestamp` must contain the timestamp extracted from the log line, event.created when the log line is read. The same could apply to package capturing where @timestamp contains the timestamp extracted from the network package and event.created when the event was created. In case the two timestamps are identical, @timestamp should be used.', + 'IP address of the destination.\n\nCan be one or multiple IPv4 or IPv6 addresses.', }, { - name: 'start', + name: 'mac', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'MAC address of the destination.', + }, + { + name: 'nat.ip', level: 'extended', - type: 'date', + type: 'ip', description: - 'event.start contains the date when the event started or when the activity was first observed.', + 'Translated ip of destination based NAT sessions (e.g. internet\nto private DMZ)\n\nTypically used with load balancers, firewalls, or routers.', }, { - name: 'end', + name: 'nat.port', level: 'extended', - type: 'date', + type: 'long', + format: 'string', description: - 'event.end contains the date when the event ended or when the activity was last observed.', + 'Port the source session is translated to by NAT Device.\n\nTypically used with load balancers, firewalls, or routers.', }, { - name: 'risk_score', + name: 'packets', level: 'core', - type: 'float', - description: - "Risk score or priority of the event (e.g. security solutions). Use your system's original value here. ", + type: 'long', + description: 'Packets sent from the destination to the source.', + example: 12, }, { - name: 'risk_score_norm', + name: 'port', + level: 'core', + type: 'long', + format: 'string', + description: 'Port of the destination.', + }, + { + name: 'registered_domain', level: 'extended', - type: 'float', + type: 'keyword', + ignore_above: 1024, description: - 'Normalized risk score or priority of the event, on a scale of 0 to 100. This is mainly useful if you use more than one system that assigns risk scores, and you want to see a normalized value across all systems.', + 'The highest registered destination domain, stripped of the subdomain.\n\nFor example, the registered domain for "foo.google.com" is "google.com".\n\nThis value can be determined precisely with a list like the public suffix\nlist (http://publicsuffix.org). Trying to approximate this by simply taking\nthe last two labels will not work well for TLDs such as "co.uk".', + example: 'google.com', }, - ], - }, - { - name: 'file', - group: 2, - title: 'File', - description: - 'A file is defined as a set of information that has been created on, or has existed on a filesystem. File objects can be associated with host events, network events, and/or file events (e.g., those produced by File Integrity Monitoring [FIM] products or services). File fields provide details about the affected file associated with the event or metric.', - type: 'group', - fields: [ { - name: 'path', + name: 'top_level_domain', level: 'extended', type: 'keyword', - description: 'Path to the file.', + ignore_above: 1024, + description: + 'The effective top level domain (eTLD), also known as the domain\nsuffix, is the last part of the domain name. For example, the top level domain\nfor google.com is "com".\n\nThis value can be determined precisely with a list like the public suffix\nlist (http://publicsuffix.org). Trying to approximate this by simply taking\nthe last label will not work well for effective TLDs such as "co.uk".', + example: 'co.uk', }, { - name: 'target_path', + name: 'user.domain', level: 'extended', type: 'keyword', - description: 'Target path for symlinks.', + ignore_above: 1024, + description: + 'Name of the directory the user is a member of.\n\nFor example, an LDAP or Active Directory domain name.', }, { - name: 'extension', + name: 'user.email', level: 'extended', type: 'keyword', - description: 'File extension. This should allow easy filtering by file extensions.', - example: 'png', + ignore_above: 1024, + description: 'User email address.', }, { - name: 'type', + name: 'user.full_name', level: 'extended', type: 'keyword', - description: 'File type (file, dir, or symlink).', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: "User's full name, if available.", + example: 'Albert Einstein', }, { - name: 'device', + name: 'user.group.domain', level: 'extended', type: 'keyword', - description: 'Device that is the source of the file.', + ignore_above: 1024, + description: + 'Name of the directory the group is a member of.\n\nFor example, an LDAP or Active Directory domain name.', }, { - name: 'inode', + name: 'user.group.id', level: 'extended', type: 'keyword', - description: 'Inode representing the file in the filesystem.', + ignore_above: 1024, + description: 'Unique identifier for the group on the system/platform.', }, { - name: 'uid', + name: 'user.group.name', level: 'extended', type: 'keyword', - description: 'The user ID (UID) or security identifier (SID) of the file owner.', + ignore_above: 1024, + description: 'Name of the group.', }, { - name: 'owner', + name: 'user.hash', level: 'extended', type: 'keyword', - description: "File owner's username.", + ignore_above: 1024, + description: + 'Unique user hash to correlate information for a user in anonymized\nform.\n\nUseful if `user.id` or `user.name` contain confidential information and cannot\nbe used.', }, { - name: 'gid', - level: 'extended', + name: 'user.id', + level: 'core', type: 'keyword', - description: 'Primary group ID (GID) of the file.', + ignore_above: 1024, + description: 'Unique identifiers of the user.', }, { - name: 'group', - level: 'extended', + name: 'user.name', + level: 'core', type: 'keyword', - description: 'Primary group name of the file.', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Short name or login of the user.', + example: 'albert', + }, + ], + }, + { + name: 'dll', + title: 'DLL', + group: 2, + description: + 'These fields contain information about code libraries dynamically\nloaded into processes.\n\n\nMany operating systems refer to "shared code libraries" with different names,\nbut this field set refers to all of the following:\n\n* Dynamic-link library (`.dll`) commonly used on Windows\n\n* Shared Object (`.so`) commonly used on Unix-like operating systems\n\n* Dynamic library (`.dylib`) commonly used on macOS', + type: 'group', + fields: [ + { + name: 'code_signature.exists', + level: 'core', + type: 'boolean', + description: 'Boolean to capture if a signature is present.', + example: 'true', + default_field: false, }, { - name: 'mode', + name: 'code_signature.status', level: 'extended', type: 'keyword', - example: 416, - description: 'Mode of the file in octal representation.', + ignore_above: 1024, + description: + 'Additional information about the certificate status.\n\nThis is useful for logging cryptographic errors with the certificate validity\nor trust status. Leave unpopulated if the validity or trust of the certificate\nwas unchecked.', + example: 'ERROR_UNTRUSTED_ROOT', + default_field: false, }, { - name: 'size', - level: 'extended', - type: 'long', - format: 'bytes', - description: 'File size in bytes (field is only added when `type` is `file`).', + name: 'code_signature.subject_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Subject name of the code signer', + example: 'Microsoft Corporation', + default_field: false, }, { - name: 'mtime', + name: 'code_signature.trusted', level: 'extended', - type: 'date', - description: 'Last time file content was modified.', + type: 'boolean', + description: + 'Stores the trust status of the certificate chain.\n\nValidating the trust of the certificate chain may be complicated, and this\nfield should only be populated by tools that actively check the status.', + example: 'true', + default_field: false, }, { - name: 'ctime', + name: 'code_signature.valid', level: 'extended', - type: 'date', - description: 'Last time file metadata changed.', + type: 'boolean', + description: + 'Boolean to capture if the digital signature is verified against\nthe binary content.\n\nLeave unpopulated if a certificate was unchecked.', + example: 'true', + default_field: false, }, - ], - }, - { - name: 'group', - title: 'Group', - group: 2, - description: - 'The group fields are meant to represent groups that are relevant to the event.', - type: 'group', - fields: [ { - name: 'id', + name: 'hash.md5', level: 'extended', type: 'keyword', - description: 'Unique identifier for the group on the system/platform.', + ignore_above: 1024, + description: 'MD5 hash.', + default_field: false, }, { - name: 'name', + name: 'hash.sha1', level: 'extended', type: 'keyword', - description: 'Name of the group.', + ignore_above: 1024, + description: 'SHA1 hash.', + default_field: false, }, - ], - }, - { - name: 'host', - title: 'Host', - group: 2, - description: - 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or on which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.', - type: 'group', - fields: [ { - name: 'hostname', - level: 'core', + name: 'hash.sha256', + level: 'extended', type: 'keyword', - description: - 'Hostname of the host. It normally contains what the `hostname` command returns on the host machine.', + ignore_above: 1024, + description: 'SHA256 hash.', + default_field: false, }, { - name: 'name', - level: 'core', + name: 'hash.sha512', + level: 'extended', type: 'keyword', - description: - 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.', + ignore_above: 1024, + description: 'SHA512 hash.', + default_field: false, }, { - name: 'id', + name: 'name', level: 'core', type: 'keyword', + ignore_above: 1024, description: - 'Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.', + 'Name of the library.\n\nThis generally maps to the name of the file on disk.', + example: 'kernel32.dll', + default_field: false, }, { - name: 'ip', - level: 'core', - type: 'ip', - description: 'Host ip address.', + name: 'path', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Full file path of the library.', + example: 'C:\\Windows\\System32\\kernel32.dll', + default_field: false, }, { - name: 'mac', - level: 'core', + name: 'pe.company', + level: 'extended', type: 'keyword', - description: 'Host mac address.', + ignore_above: 1024, + description: 'Internal company name of the file, provided at compile-time.', + example: 'Microsoft Corporation', + default_field: false, }, { - name: 'type', - level: 'core', + name: 'pe.description', + level: 'extended', type: 'keyword', - description: - 'Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.', + ignore_above: 1024, + description: 'Internal description of the file, provided at compile-time.', + example: 'Paint', + default_field: false, }, { - name: 'architecture', - level: 'core', + name: 'pe.file_version', + level: 'extended', type: 'keyword', - example: 'x86_64', - description: 'Operating system architecture.', + ignore_above: 1024, + description: 'Internal version of the file, provided at compile-time.', + example: '6.3.9600.17415', + default_field: false, }, { - name: 'os', - title: 'Operating System', - group: 2, - description: 'The OS fields contain information about the operating system.', - reusable: { - top_level: false, - expected: ['observer', 'host', 'user_agent'], - }, - type: 'group', - fields: [ - { - name: 'platform', - level: 'extended', - type: 'keyword', - description: 'Operating system platform (such centos, ubuntu, windows).', - example: 'darwin', - }, - { - name: 'name', - level: 'extended', - type: 'keyword', - example: 'Mac OS X', - description: 'Operating system name, without the version.', - }, - { - name: 'full', - level: 'extended', - type: 'keyword', - example: 'Mac OS Mojave', - description: 'Operating system name, including the version or code name.', - }, - { - name: 'family', - level: 'extended', - type: 'keyword', - example: 'debian', - description: 'OS family (such as redhat, debian, freebsd, windows).', - }, - { - name: 'version', - level: 'extended', - type: 'keyword', - example: '10.14.1', - description: 'Operating system version as a raw string.', - }, - { - name: 'kernel', - level: 'extended', - type: 'keyword', - example: '4.4.0-112-generic', - description: 'Operating system kernel version as a raw string.', - }, - ], + name: 'pe.original_file_name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal name of the file, provided at compile-time.', + example: 'MSPAINT.EXE', + default_field: false, }, { - name: 'geo', - title: 'Geo', - group: 2, - description: - 'Geo fields can carry data about a specific location related to an event or geo information derived from an IP field.', - type: 'group', - fields: [ - { - name: 'location', - level: 'core', - type: 'geo_point', - description: 'Longitude and latitude.', - example: '{ "lon": -73.614830, "lat": 45.505918 }', - }, - { - name: 'continent_name', - level: 'core', - type: 'keyword', - description: 'Name of the continent.', - example: 'North America', - }, - { - name: 'country_name', - level: 'core', - type: 'keyword', - description: 'Country name.', - example: 'Canada', - }, - { - name: 'region_name', - level: 'core', - type: 'keyword', - description: 'Region name.', - example: 'Quebec', - }, - { - name: 'city_name', - level: 'core', - type: 'keyword', - description: 'City name.', - example: 'Montreal', - }, - { - name: 'country_iso_code', - level: 'core', - type: 'keyword', - description: 'Country ISO code.', - example: 'CA', - }, - { - name: 'region_iso_code', - level: 'core', - type: 'keyword', - description: 'Region ISO code.', - example: 'CA-QC', - }, - { - name: 'name', - level: 'extended', - type: 'keyword', - description: - 'User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation.', - example: 'boston-dc', - }, - ], + name: 'pe.product', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal product name of the file, provided at compile-time.', + example: 'Microsoft® Windows® Operating System', + default_field: false, }, ], }, { - name: 'http', - title: 'HTTP', + name: 'dns', + title: 'DNS', group: 2, - description: 'Fields related to HTTP activity.', + description: + 'Fields describing DNS queries and answers.\n\nDNS events should either represent a single DNS query prior to getting answers\n(`dns.type:query`) or they should represent a full exchange and contain the\nquery details as well as all of the answers that were provided for this query\n(`dns.type:answer`).', type: 'group', fields: [ { - name: 'request.method', + name: 'answers', level: 'extended', - type: 'keyword', + type: 'object', + object_type: 'keyword', description: - 'Http request method. The field value must be normalized to lowercase for querying. See "Lowercase Capitalization" in the "Implementing ECS" section.', - example: 'get, post, put', + 'An array containing an object for each answer section returned\nby the server.\n\nThe main keys that should be present in these objects are defined by ECS.\nRecords that have more information may contain more keys than what ECS defines.\n\nNot all DNS data sources give all details about DNS answers. At minimum, answer\nobjects must contain the `data` key. If more information is available, map\nas much of it to ECS as possible, and add any additional fields to the answer\nobjects as custom fields.', }, { - name: 'request.body.content', + name: 'answers.class', level: 'extended', type: 'keyword', - description: 'The full http request body.', - example: 'Hello world', + ignore_above: 1024, + description: 'The class of DNS data contained in this resource record.', + example: 'IN', }, { - name: 'request.referrer', + name: 'answers.data', level: 'extended', type: 'keyword', - description: 'Referrer for this HTTP request.', - example: 'https://blog.example.com/', + ignore_above: 1024, + description: + 'The data describing the resource.\n\nThe meaning of this data depends on the type and class of the resource record.', + example: '10.10.10.10', }, { - name: 'response.status_code', + name: 'answers.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The domain name to which this resource record pertains.\n\nIf a chain of CNAME is being resolved, each answer `name` should be the\none that corresponds with the answer `data`. It should not simply be the\noriginal `question.name` repeated.', + example: 'www.google.com', + }, + { + name: 'answers.ttl', level: 'extended', type: 'long', - description: 'Http response status code.', - example: 404, + description: + 'The time interval in seconds that this resource record may be cached\nbefore it should be discarded. Zero values mean that the data should not be\ncached.', + example: 180, }, { - name: 'response.body.content', + name: 'answers.type', level: 'extended', type: 'keyword', - description: 'The full http response body.', - example: 'Hello world', + ignore_above: 1024, + description: 'The type of data contained in this resource record.', + example: 'CNAME', }, { - name: 'version', + name: 'header_flags', level: 'extended', type: 'keyword', - description: 'Http version.', - example: 1.1, + ignore_above: 1024, + description: + 'Array of 2 letter DNS header flags.\n\nExpected values are: AA, TC, RD, RA, AD, CD, DO.', + example: ['RD', 'RA'], }, { - name: 'request.bytes', + name: 'id', level: 'extended', - type: 'long', - format: 'bytes', - description: 'Total size in bytes of the request (body and headers).', - example: 1437, + type: 'keyword', + ignore_above: 1024, + description: + 'The DNS packet identifier assigned by the program that generated\nthe query. The identifier is copied to the response.', + example: 62111, }, { - name: 'request.body.bytes', + name: 'op_code', level: 'extended', - type: 'long', - format: 'bytes', - description: 'Size in bytes of the request body.', - example: 887, + type: 'keyword', + ignore_above: 1024, + description: + 'The DNS operation code that specifies the kind of query in the\nmessage. This value is set by the originator of a query and copied into the\nresponse.', + example: 'QUERY', }, { - name: 'response.bytes', + name: 'question.class', level: 'extended', - type: 'long', - format: 'bytes', - description: 'Total size in bytes of the response (body and headers).', - example: 1437, + type: 'keyword', + ignore_above: 1024, + description: 'The class of records being queried.', + example: 'IN', }, { - name: 'response.body.bytes', + name: 'question.name', level: 'extended', - type: 'long', - format: 'bytes', - description: 'Size in bytes of the response body.', - example: 887, + type: 'keyword', + ignore_above: 1024, + description: + 'The name being queried.\n\nIf the name field contains non-printable characters (below 32 or above 126),\nthose characters should be represented as escaped base 10 integers (\\DDD).\nBack slashes and quotes should be escaped. Tabs, carriage returns, and line\nfeeds should be converted to \\t, \\r, and \\n respectively.', + example: 'www.google.com', + }, + { + name: 'question.registered_domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The highest registered domain, stripped of the subdomain.\n\nFor example, the registered domain for "foo.google.com" is "google.com".\n\nThis value can be determined precisely with a list like the public suffix\nlist (http://publicsuffix.org). Trying to approximate this by simply taking\nthe last two labels will not work well for TLDs such as "co.uk".', + example: 'google.com', + }, + { + name: 'question.subdomain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The subdomain is all of the labels under the registered_domain.\n\nIf the domain has multiple levels of subdomain, such as "sub2.sub1.example.com",\nthe subdomain field should contain "sub2.sub1", with no trailing period.', + example: 'www', + }, + { + name: 'question.top_level_domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The effective top level domain (eTLD), also known as the domain\nsuffix, is the last part of the domain name. For example, the top level domain\nfor google.com is "com".\n\nThis value can be determined precisely with a list like the public suffix\nlist (http://publicsuffix.org). Trying to approximate this by simply taking\nthe last label will not work well for effective TLDs such as "co.uk".', + example: 'co.uk', + }, + { + name: 'question.type', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'The type of record being queried.', + example: 'AAAA', + }, + { + name: 'resolved_ip', + level: 'extended', + type: 'ip', + description: + 'Array containing all IPs seen in `answers.data`.\n\nThe `answers` array can be difficult to use, because of the variety of data\nformats it can contain. Extracting all IP addresses seen in there to `dns.resolved_ip`\nmakes it possible to index them as IP addresses, and makes them easier to\nvisualize and query for.', + example: ['10.10.10.10', '10.10.10.11'], + }, + { + name: 'response_code', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'The DNS response code.', + example: 'NOERROR', + }, + { + name: 'type', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The type of DNS event captured, query or answer.\n\nIf your source of DNS events only gives you DNS queries, you should only create\ndns events of type `dns.type:query`.\n\nIf your source of DNS events gives you answers as well, you should create\none event per query (optionally as soon as the query is seen). And a second\nevent containing all query details as well as an array of answers.', + example: 'answer', }, ], }, { - name: 'log', - title: 'Log', - description: 'Fields which are specific to log events.', + name: 'ecs', + title: 'ECS', + group: 2, + description: 'Meta-information specific to ECS.', type: 'group', fields: [ { - name: 'level', + name: 'version', + level: 'core', + required: true, + type: 'keyword', + ignore_above: 1024, + description: + 'ECS version this event conforms to. `ecs.version` is a required\nfield and must exist in all events.\n\nWhen querying across multiple indices -- which may conform to slightly different\nECS versions -- this field lets integrations adjust to the schema version\nof the events.', + example: '1.0.0', + }, + ], + }, + { + name: 'error', + title: 'Error', + group: 2, + description: + 'These fields can represent errors of any kind.\n\nUse them for errors that happen while fetching events or in cases where the\nevent itself contains an error.', + type: 'group', + fields: [ + { + name: 'code', level: 'core', type: 'keyword', - description: 'Log level of the log event. Some examples are `WARN`, `ERR`, `INFO`.', - example: 'ERR', + ignore_above: 1024, + description: 'Error code describing the error.', }, { - name: 'original', + name: 'id', level: 'core', type: 'keyword', - example: 'Sep 19 08:26:10 localhost My log', - index: false, - doc_values: false, - description: - " This is the original log message and contains the full log message before splitting it up in multiple parts. In contrast to the `message` field which can contain an extracted part of the log message, this field contains the original, full log message. It can have already some modifications applied like encoding or new lines removed to clean up the log message. This field is not indexed and doc_values are disabled so it can't be queried but the value can be retrieved from `_source`. ", + ignore_above: 1024, + description: 'Unique identifier for the error.', + }, + { + name: 'message', + level: 'core', + type: 'text', + description: 'Error message.', + }, + { + name: 'stack_trace', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'The stack trace of this error in plain text.', + }, + { + name: 'type', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'The type of the error, for example the class name of the exception.', + example: 'java.lang.NullPointerException', }, ], }, { - name: 'network', - title: 'Network', + name: 'event', + title: 'Event', group: 2, description: - 'The network is defined as the communication path over which a host or network event happens. The network.* fields should be populated with details about the network activity associated with an event.', + 'The event fields are used for context information about the log\nor metric event itself.\n\nA log is defined as an event containing details of something that happened.\nLog events must include the time at which the thing happened. Examples of log\nevents include a process starting on a host, a network packet being sent from\na source to a destination, or a network connection between a client and a server\nbeing initiated or closed. A metric is defined as an event containing one or\nmore numerical measurements and the time at which the measurement was taken.\nExamples of metric events include memory pressure measured on a host and device\ntemperature. See the `event.kind` definition in this section for additional\ndetails about metric and state events.', type: 'group', fields: [ { - name: 'name', - level: 'extended', + name: 'action', + level: 'core', type: 'keyword', - description: 'Name given by operators to sections of their network.', - example: 'Guest Wifi', + ignore_above: 1024, + description: + 'The action captured by the event.\n\nThis describes the information in the event. It is more specific than `event.category`.\nExamples are `group-add`, `process-started`, `file-created`. The value is\nnormally defined by the implementer.', + example: 'user-password-change', }, { - name: 'type', + name: 'category', level: 'core', type: 'keyword', + ignore_above: 1024, description: - 'In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc The field value must be normalized to lowercase for querying. See "Lowercase Capitalization" in the "Implementing ECS" section.', - example: 'ipv4', + 'This is one of four ECS Categorization Fields, and indicates the\nsecond level in the ECS category hierarchy.\n\n`event.category` represents the "big buckets" of ECS categories. For example,\nfiltering on `event.category:process` yields all events relating to process\nactivity. This field is closely related to `event.type`, which is used as\na subcategory.\n\nThis field is an array. This will allow proper categorization of some events\nthat fall in multiple categories.', + example: 'authentication', }, { - name: 'iana_number', + name: 'code', level: 'extended', type: 'keyword', + ignore_above: 1024, description: - 'IANA Protocol Number (https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml). Standardized list of protocols. This aligns well with NetFlow and sFlow related logs which use the IANA Protocol Number.', - example: 6, + 'Identification code for this event, if one exists.\n\nSome event sources use event codes to identify messages unambiguously, regardless\nof message language or wording adjustments over time. An example of this is\nthe Windows Event ID.', + example: 4648, }, { - name: 'transport', + name: 'created', + level: 'core', + type: 'date', + description: + 'event.created contains the date/time when the event was first\nread by an agent, or by your pipeline.\n\nThis field is distinct from @timestamp in that @timestamp typically contain\nthe time extracted from the original event.\n\nIn most situations, these two timestamps will be slightly different. The difference\ncan be used to calculate the delay between your source generating an event,\nand the time when your agent first processed it. This can be used to monitor\nyour agent or pipeline ability to keep up with your event source.\n\nIn case the two timestamps are identical, @timestamp should be used.', + example: '2016-05-23T08:05:34.857Z', + }, + { + name: 'dataset', level: 'core', type: 'keyword', + ignore_above: 1024, description: - 'Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. See "Lowercase Capitalization" in the "Implementing ECS" section.', - example: 'tcp', + 'Name of the dataset.\n\nIf an event source publishes more than one type of log or events (e.g. access\nlog, error log), the dataset is used to specify which one the event comes\nfrom.\n\nIt is recommended but not required to start the dataset name with the module\nname, followed by a dot, then the dataset name.', + example: 'apache.access', }, { - name: 'application', + name: 'duration', + level: 'core', + type: 'long', + format: 'duration', + input_format: 'nanoseconds', + output_format: 'asMilliseconds', + output_precision: 1, + description: + 'Duration of the event in nanoseconds.\n\nIf event.start and event.end are known this value should be the difference\nbetween the end and start time.', + }, + { + name: 'end', + level: 'extended', + type: 'date', + description: + 'event.end contains the date when the event ended or when the activity\nwas last observed.', + }, + { + name: 'hash', level: 'extended', type: 'keyword', + ignore_above: 1024, description: - 'A name given to an application. This can be arbitrarily assigned for things like microservices, but also apply to things like skype, icq, facebook, twitter. This would be used in situations where the vendor or service can be decoded such as from the source/dest IP owners, ports, or wire format. The field value must be normalized to lowercase for querying. See "Lowercase Capitalization" in the "Implementing ECS" section.', - example: 'aim', + 'Hash (perhaps logstash fingerprint) of raw field to be able to\ndemonstrate log integrity.', + example: '123456789012345678901234567890ABCD', }, { - name: 'protocol', + name: 'id', level: 'core', type: 'keyword', + ignore_above: 1024, + description: 'Unique ID to describe the event.', + example: '8a4f500d', + }, + { + name: 'ingested', + level: 'core', + type: 'date', description: - 'L7 Network protocol name. ex. http, lumberjack, transport protocol. The field value must be normalized to lowercase for querying. See "Lowercase Capitalization" in the "Implementing ECS" section.', - example: 'http', + 'Timestamp when an event arrived in the central data store.\n\nThis is different from `@timestamp`, which is when the event originally occurred. It is\nalso different from `event.created`, which is meant to capture the first time\nan agent saw the event.\n\nIn normal conditions, assuming no tampering, the timestamps should chronologically\nlook like this: `@timestamp` < `event.created` < `event.ingested`.', + example: '2016-05-23T08:05:35.101Z', + default_field: false, }, { - name: 'direction', + name: 'kind', level: 'core', type: 'keyword', + ignore_above: 1024, description: - "Direction of the network traffic. Recommended values are: * inbound * outbound * internal * external * unknown When mapping events from a host-based monitoring context, populate this field from the host's point of view. When mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of your network perimeter. ", - example: 'inbound', + 'This is one of four ECS Categorization Fields, and indicates the\nhighest level in the ECS category hierarchy.\n\n`event.kind` gives high-level information about what type of information the\nevent contains, without being specific to the contents of the event. For example,\nvalues of this field distinguish alert events from metric events.\n\nThe value of this field can be used to inform how these kinds of events should\nbe handled. They may warrant different retention, different access control,\nit may also help understand whether the data coming in at a regular interval\nor not.', + example: 'alert', }, { - name: 'forwarded_ip', + name: 'module', level: 'core', - type: 'ip', - description: 'Host IP address when the source IP address is the proxy.', - example: '192.1.1.2', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the module this data is coming from.\n\nIf your monitoring agent supports the concept of modules or plugins to process\nevents of a given source (e.g. Apache logs), `event.module` should contain\nthe name of this module.', + example: 'apache', }, { - name: 'community_id', + name: 'original', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'Raw text message of entire event. Used to demonstrate log integrity.\n\nThis field is not indexed and doc_values are disabled. It cannot be searched,\nbut it can be retrieved from `_source`.', + example: + 'Sep 19 08:26:10 host CEF:0|Security| threatmanager|1.0|100|\nworm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2spt=1232', + }, + { + name: 'outcome', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'This is one of four ECS Categorization Fields, and indicates the\nlowest level in the ECS category hierarchy.\n\n`event.outcome` simply denotes whether the event represents a success or a\nfailure from the perspective of the entity that produced the event.\n\nNote that when a single transaction is described in multiple events, each\nevent may populate different values of `event.outcome`, according to their\nperspective.\n\nAlso note that in the case of a compound event (a single event that contains\nmultiple logical events), this field should be populated with the value that\nbest captures the overall success or failure from the perspective of the event\nproducer.\n\nFurther note that not all events will have an associated outcome. For example,\nthis field is generally not populated for metric events, events with `event.type:info`,\nor any events for which an outcome does not make logical sense.', + example: 'success', + }, + { + name: 'provider', level: 'extended', type: 'keyword', + ignore_above: 1024, description: - 'A hash of source and destination IPs and ports, as well as the protocol used in a communication. This is a tool-agnostic standard to identify flows. Learn more at https://github.com/corelight/community-id-spec.', - example: '1:hO+sN4H+MG5MY/8hIrXPqc4ZQz0=', + 'Source of the event.\n\nEvent transports such as Syslog or the Windows Event Log typically mention\nthe source of an event. It can be the name of the software that generated\nthe event (e.g. Sysmon, httpd), or of a subsystem of the operating system\n(kernel, Microsoft-Windows-Security-Auditing).', + example: 'kernel', }, { - name: 'bytes', + name: 'reference', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Reference URL linking to additional information about this event.\n\nThis URL links to a static definition of the this event. Alert events, indicated\nby `event.kind:alert`, are a common use case for this field.', + example: 'https://system.vendor.com/event/#0001234', + default_field: false, + }, + { + name: 'risk_score', level: 'core', + type: 'float', + description: + "Risk score or priority of the event (e.g. security solutions).\nUse your system's original value here.", + }, + { + name: 'risk_score_norm', + level: 'extended', + type: 'float', + description: + 'Normalized risk score or priority of the event, on a scale of\n0 to 100.\n\nThis is mainly useful if you use more than one system that assigns risk scores,\nand you want to see a normalized value across all systems.', + }, + { + name: 'sequence', + level: 'extended', type: 'long', - format: 'bytes', + format: 'string', description: - 'Total bytes transferred in both directions. If `source.bytes` and `destination.bytes` are known, `network.bytes` is their sum.', - example: 368, + 'Sequence number of the event.\n\nThe sequence number is a value published by some event sources, to make the\nexact ordering of events unambiguous, regardless of the timestamp precision.', }, { - name: 'packets', + name: 'severity', level: 'core', type: 'long', + format: 'string', description: - 'Total packets transferred in both directions. If `source.packets` and `destination.packets` are known, `network.packets` is their sum.', - example: 24, + 'The numeric severity of the event according to your event source.\n\nWhat the different severity values mean can be different between sources and\nuse cases. It is up to the implementer to make sure severities are consistent\nacross events from the same source.\n\nThe Syslog severity belongs in `log.syslog.severity.code`. `event.severity`\nis meant to represent the severity according to the event source (e.g. firewall,\nIDS). If the event source does not publish its own severity, you may optionally\ncopy the `log.syslog.severity.code` to `event.severity`.', + example: 7, + }, + { + name: 'start', + level: 'extended', + type: 'date', + description: + 'event.start contains the date when the event started or when the\nactivity was first observed.', + }, + { + name: 'timezone', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'This field should be populated when the event timestamp does\nnot include timezone information already (e.g. default Syslog timestamps).\nIt is optional otherwise.\n\nAcceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"),\nabbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00").', + }, + { + name: 'type', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'This is one of four ECS Categorization Fields, and indicates the\nthird level in the ECS category hierarchy.\n\n`event.type` represents a categorization "sub-bucket" that, when used along\nwith the `event.category` field values, enables filtering events down to a\nlevel appropriate for single visualization.\n\nThis field is an array. This will allow proper categorization of some events\nthat fall in multiple event types.', + }, + { + name: 'url', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'URL linking to an external system to continue investigation of\nthis event.\n\nThis URL links to another system where in-depth investigation of the specific\noccurence of this event can take place. Alert events, indicated by `event.kind:alert`,\nare a common use case for this field.', + example: 'https://mysystem.mydomain.com/alert/5271dedb-f5b0-4218-87f0-4ac4870a38fe', + default_field: false, }, ], }, { - name: 'observer', - title: 'Observer', + name: 'file', + title: 'File', group: 2, description: - 'An observer is defined as a special network, security, or application device used to detect, observe, or create network, security, or application-related events and metrics. This could be a custom hardware appliance or a server that has been configured to run special network, security, or application software. Examples include firewalls, intrusion detection/prevention systems, network monitoring sensors, web application firewalls, data loss prevention systems, and APM servers. The observer.* fields shall be populated with details of the system, if any, that detects, observes and/or creates a network, security, or application event or metric. Message queues and ETL components used in processing events or metrics are not considered observers in ECS.', + 'A file is defined as a set of information that has been created\non, or has existed on a filesystem.\n\nFile objects can be associated with host events, network events, and/or file\nevents (e.g., those produced by File Integrity Monitoring [FIM] products or\nservices). File fields provide details about the affected file associated with\nthe event or metric.', type: 'group', fields: [ { - name: 'mac', - level: 'core', - type: 'keyword', - description: 'MAC address of the observer', + name: 'accessed', + level: 'extended', + type: 'date', + description: + 'Last time the file was accessed.\n\nNote that not all filesystems keep track of access time.', }, { - name: 'ip', - level: 'core', - type: 'ip', - description: 'IP address of the observer.', + name: 'attributes', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Array of file attributes.\n\nAttributes names will vary by platform. Here is a non-exhaustive list of values\nthat are expected in this field: archive, compressed, directory, encrypted,\nexecute, hidden, read, readonly, system, write.', + example: '["readonly", "system"]', + default_field: false, }, { - name: 'hostname', + name: 'code_signature.exists', level: 'core', - type: 'keyword', - description: 'Hostname of the observer.', + type: 'boolean', + description: 'Boolean to capture if a signature is present.', + example: 'true', + default_field: false, }, { - name: 'vendor', - level: 'core', + name: 'code_signature.status', + level: 'extended', type: 'keyword', - description: 'observer vendor information.', + ignore_above: 1024, + description: + 'Additional information about the certificate status.\n\nThis is useful for logging cryptographic errors with the certificate validity\nor trust status. Leave unpopulated if the validity or trust of the certificate\nwas unchecked.', + example: 'ERROR_UNTRUSTED_ROOT', + default_field: false, }, { - name: 'version', + name: 'code_signature.subject_name', level: 'core', type: 'keyword', - description: 'Observer version.', + ignore_above: 1024, + description: 'Subject name of the code signer', + example: 'Microsoft Corporation', + default_field: false, }, { - name: 'serial_number', + name: 'code_signature.trusted', level: 'extended', - type: 'keyword', - description: 'Observer serial number.', - }, - { - name: 'type', - level: 'core', - type: 'keyword', + type: 'boolean', description: - 'The type of the observer the data is coming from. There is no predefined list of observer types. Some examples are `forwarder`, `firewall`, `ids`, `ips`, `proxy`, `poller`, `sensor`, `APM server`.', - example: 'firewall', + 'Stores the trust status of the certificate chain.\n\nValidating the trust of the certificate chain may be complicated, and this\nfield should only be populated by tools that actively check the status.', + example: 'true', + default_field: false, }, { - name: 'os', - title: 'Operating System', - group: 2, - description: 'The OS fields contain information about the operating system.', - reusable: { - top_level: false, - expected: ['observer', 'host', 'user_agent'], - }, - type: 'group', - fields: [ - { - name: 'platform', - level: 'extended', - type: 'keyword', - description: 'Operating system platform (such centos, ubuntu, windows).', - example: 'darwin', - }, - { - name: 'name', - level: 'extended', - type: 'keyword', - example: 'Mac OS X', - description: 'Operating system name, without the version.', - }, - { - name: 'full', - level: 'extended', - type: 'keyword', - example: 'Mac OS Mojave', - description: 'Operating system name, including the version or code name.', - }, - { - name: 'family', - level: 'extended', - type: 'keyword', - example: 'debian', - description: 'OS family (such as redhat, debian, freebsd, windows).', - }, - { - name: 'version', - level: 'extended', - type: 'keyword', - example: '10.14.1', - description: 'Operating system version as a raw string.', - }, - { - name: 'kernel', - level: 'extended', - type: 'keyword', - example: '4.4.0-112-generic', - description: 'Operating system kernel version as a raw string.', - }, - ], + name: 'code_signature.valid', + level: 'extended', + type: 'boolean', + description: + 'Boolean to capture if the digital signature is verified against\nthe binary content.\n\nLeave unpopulated if a certificate was unchecked.', + example: 'true', + default_field: false, }, { - name: 'geo', - title: 'Geo', - group: 2, + name: 'created', + level: 'extended', + type: 'date', description: - 'Geo fields can carry data about a specific location related to an event or geo information derived from an IP field.', - type: 'group', - fields: [ - { - name: 'location', - level: 'core', - type: 'geo_point', - description: 'Longitude and latitude.', - example: '{ "lon": -73.614830, "lat": 45.505918 }', - }, - { - name: 'continent_name', - level: 'core', - type: 'keyword', - description: 'Name of the continent.', - example: 'North America', - }, - { - name: 'country_name', - level: 'core', - type: 'keyword', - description: 'Country name.', - example: 'Canada', - }, - { - name: 'region_name', - level: 'core', - type: 'keyword', - description: 'Region name.', - example: 'Quebec', - }, - { - name: 'city_name', - level: 'core', - type: 'keyword', - description: 'City name.', - example: 'Montreal', - }, - { - name: 'country_iso_code', - level: 'core', - type: 'keyword', - description: 'Country ISO code.', - example: 'CA', - }, - { - name: 'region_iso_code', - level: 'core', - type: 'keyword', - description: 'Region ISO code.', - example: 'CA-QC', - }, - { - name: 'name', - level: 'extended', - type: 'keyword', - description: - 'User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation.', - example: 'boston-dc', - }, - ], + 'File creation time.\n\nNote that not all filesystems store the creation time.', }, - ], - }, - { - name: 'organization', - title: 'Organization', - group: 2, - description: - 'The organization fields enrich data with information about the company or entity the data is associated with. These fields help you arrange or filter data stored in an index by one or multiple organizations.', - type: 'group', - fields: [ { - name: 'name', + name: 'ctime', + level: 'extended', + type: 'date', + description: + 'Last time the file attributes or metadata changed.\n\nNote that changes to the file content will update `mtime`. This implies `ctime`\nwill be adjusted at the same time, since `mtime` is an attribute of the file.', + }, + { + name: 'device', level: 'extended', type: 'keyword', - description: 'Organization name.', + ignore_above: 1024, + description: 'Device that is the source of the file.', + example: 'sda', }, { - name: 'id', + name: 'directory', level: 'extended', type: 'keyword', - description: 'Unique identifier for the organization.', + ignore_above: 1024, + description: + 'Directory where the file is located. It should include the drive\nletter, when appropriate.', + example: '/home/alice', }, - ], - }, - { - name: 'os', - title: 'Operating System', - group: 2, - description: 'The OS fields contain information about the operating system.', - reusable: { - top_level: false, - expected: ['observer', 'host', 'user_agent'], - }, - type: 'group', - fields: [ { - name: 'platform', + name: 'drive_letter', level: 'extended', type: 'keyword', - description: 'Operating system platform (such centos, ubuntu, windows).', - example: 'darwin', + ignore_above: 1, + description: + 'Drive letter where the file is located. This field is only relevant\non Windows.\n\nThe value should be uppercase, and not include the colon.', + example: 'C', + default_field: false, }, { - name: 'name', + name: 'extension', level: 'extended', type: 'keyword', - example: 'Mac OS X', - description: 'Operating system name, without the version.', + ignore_above: 1024, + description: 'File extension.', + example: 'png', }, { - name: 'full', + name: 'gid', level: 'extended', type: 'keyword', - example: 'Mac OS Mojave', - description: 'Operating system name, including the version or code name.', + ignore_above: 1024, + description: 'Primary group ID (GID) of the file.', + example: '1001', }, { - name: 'family', + name: 'group', level: 'extended', type: 'keyword', - example: 'debian', - description: 'OS family (such as redhat, debian, freebsd, windows).', + ignore_above: 1024, + description: 'Primary group name of the file.', + example: 'alice', }, { - name: 'version', + name: 'hash.md5', level: 'extended', type: 'keyword', - example: '10.14.1', - description: 'Operating system version as a raw string.', + ignore_above: 1024, + description: 'MD5 hash.', }, { - name: 'kernel', + name: 'hash.sha1', level: 'extended', type: 'keyword', - example: '4.4.0-112-generic', - description: 'Operating system kernel version as a raw string.', + ignore_above: 1024, + description: 'SHA1 hash.', }, - ], - }, - { - name: 'process', - title: 'Process', - group: 2, - description: - 'These fields contain information about a process. These fields can help you correlate metrics information with a process id/name from a log message. The `process.pid` often stays in the metric itself and is copied to the global field for correlation.', - type: 'group', - fields: [ { - name: 'pid', - level: 'core', - type: 'long', - description: 'Process id.', - example: 'ssh', + name: 'hash.sha256', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'SHA256 hash.', }, { - name: 'name', + name: 'hash.sha512', level: 'extended', type: 'keyword', - description: 'Process name. Sometimes called program name or similar.', - example: 'ssh', + ignore_above: 1024, + description: 'SHA512 hash.', }, { - name: 'ppid', + name: 'inode', level: 'extended', - type: 'long', - description: 'Process parent id.', + type: 'keyword', + ignore_above: 1024, + description: 'Inode representing the file in the filesystem.', + example: '256383', }, { - name: 'args', + name: 'mime_type', level: 'extended', type: 'keyword', - description: 'Process arguments. May be filtered to protect sensitive information.', - example: ['ssh', '-l', 'user', '10.0.0.16'], + ignore_above: 1024, + description: + 'MIME type should identify the format of the file or stream of bytes\nusing https://www.iana.org/assignments/media-types/media-types.xhtml[IANA\nofficial types], where possible. When more than one type is applicable, the\nmost specific type should be used.', + default_field: false, }, { - name: 'executable', + name: 'mode', level: 'extended', type: 'keyword', - description: 'Absolute path to the process executable.', - example: '/usr/bin/ssh', + ignore_above: 1024, + description: 'Mode of the file in octal representation.', + example: '0640', }, { - name: 'title', + name: 'mtime', + level: 'extended', + type: 'date', + description: 'Last time the file content was modified.', + }, + { + name: 'name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the file including the extension, without the directory.', + example: 'example.png', + }, + { + name: 'owner', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: "File owner's username.", + example: 'alice', + }, + { + name: 'path', level: 'extended', type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], description: - 'Process title. The proctitle, some times the same as process name. Can also be different: for example a browser setting its title to the web page currently opened.', + 'Full path to the file, including the file name. It should include\nthe drive letter, when appropriate.', + example: '/home/alice/example.png', }, { - name: 'thread.id', + name: 'pe.company', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal company name of the file, provided at compile-time.', + example: 'Microsoft Corporation', + default_field: false, + }, + { + name: 'pe.description', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal description of the file, provided at compile-time.', + example: 'Paint', + default_field: false, + }, + { + name: 'pe.file_version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal version of the file, provided at compile-time.', + example: '6.3.9600.17415', + default_field: false, + }, + { + name: 'pe.original_file_name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal name of the file, provided at compile-time.', + example: 'MSPAINT.EXE', + default_field: false, + }, + { + name: 'pe.product', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal product name of the file, provided at compile-time.', + example: 'Microsoft® Windows® Operating System', + default_field: false, + }, + { + name: 'size', level: 'extended', type: 'long', - example: 4242, - description: 'Thread ID.', + description: 'File size in bytes.\n\nOnly relevant when `file.type` is "file".', + example: 16384, }, { - name: 'start', + name: 'target_path', level: 'extended', - type: 'date', - example: '2016-05-23T08:05:34.853Z', - description: 'The time the process started.', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Target path for symlinks.', }, { - name: 'working_directory', + name: 'type', level: 'extended', type: 'keyword', - example: '/home/alice', - description: 'The working directory of the process.', + ignore_above: 1024, + description: 'File type (file, dir, or symlink).', + example: 'file', }, - ], - }, - { - name: 'related', - title: 'Related', - group: 2, - description: - 'This field set is meant to facilitate pivoting around a piece of data. Some pieces of information can be seen in many places in ECS. To facilitate searching for them, append values to their corresponding field in `related.`. A concrete example is IP addresses, which can be under host, observer, source, destination, client, server, and network.forwarded_ip. If you append all IPs to `related.ip`, you can then search for a given IP trivially, no matter where it appeared, by querying `related.ip:a.b.c.d`.', - type: 'group', - fields: [ { - name: 'ip', + name: 'uid', level: 'extended', - type: 'ip', - description: 'All of the IPs seen on your event.', + type: 'keyword', + ignore_above: 1024, + description: 'The user ID (UID) or security identifier (SID) of the file owner.', + example: '1001', }, ], }, { - name: 'server', - title: 'Server', + name: 'geo', + title: 'Geo', group: 2, description: - 'A Server is defined as the responder in a network connection for events regarding sessions, connections, or bidirectional flow records. For TCP events, the server is the receiver of the initial SYN packet(s) of the TCP connection. For other protocols, the server is generally the responder in the network transaction. Some systems actually use the term "responder" to refer the server in TCP connections. The server fields describe details about the system acting as the server in the network event. Server fields are usually populated in conjunction with client fields. Server fields are generally not populated for packet-level events. Client / server representations can add semantic context to an exchange, which is helpful to visualize the data in certain situations. If your context falls in that category, you should still ensure that source and destination are filled appropriately.', + 'Geo fields can carry data about a specific location related to an\nevent.\n\nThis geolocation information can be derived from techniques such as Geo IP,\nor be user-supplied.', type: 'group', fields: [ { - name: 'address', - level: 'extended', + name: 'city_name', + level: 'core', type: 'keyword', - description: - 'Some event server addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is.', + ignore_above: 1024, + description: 'City name.', + example: 'Montreal', }, { - name: 'ip', + name: 'continent_name', level: 'core', - type: 'ip', - description: 'IP address of the server. Can be one or multiple IPv4 or IPv6 addresses.', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the continent.', + example: 'North America', }, { - name: 'port', + name: 'country_iso_code', level: 'core', - type: 'long', - description: 'Port of the server.', + type: 'keyword', + ignore_above: 1024, + description: 'Country ISO code.', + example: 'CA', }, { - name: 'mac', + name: 'country_name', level: 'core', type: 'keyword', - description: 'MAC address of the server.', + ignore_above: 1024, + description: 'Country name.', + example: 'Canada', }, { - name: 'domain', + name: 'location', level: 'core', - type: 'keyword', - description: 'Server domain.', + type: 'geo_point', + description: 'Longitude and latitude.', + example: '{ "lon": -73.614830, "lat": 45.505918 }', }, { - name: 'bytes', - level: 'core', - type: 'long', - format: 'bytes', - example: 184, - description: 'Bytes sent from the server to the client.', + name: 'name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'User-defined description of a location, at the level of granularity\nthey care about.\n\nCould be the name of their data centers, the floor number, if this describes\na local physical entity, city names.\n\nNot typically used in automated geolocation.', + example: 'boston-dc', }, { - name: 'packets', + name: 'region_iso_code', level: 'core', - type: 'long', - example: 12, - description: 'Packets sent from the server to the client.', + type: 'keyword', + ignore_above: 1024, + description: 'Region ISO code.', + example: 'CA-QC', }, { - name: 'geo', - title: 'Geo', - group: 2, - description: - 'Geo fields can carry data about a specific location related to an event or geo information derived from an IP field.', - type: 'group', - fields: [ - { - name: 'location', - level: 'core', - type: 'geo_point', - description: 'Longitude and latitude.', - example: '{ "lon": -73.614830, "lat": 45.505918 }', - }, - { - name: 'continent_name', - level: 'core', - type: 'keyword', - description: 'Name of the continent.', - example: 'North America', - }, - { - name: 'country_name', - level: 'core', - type: 'keyword', - description: 'Country name.', - example: 'Canada', - }, - { - name: 'region_name', - level: 'core', - type: 'keyword', - description: 'Region name.', - example: 'Quebec', - }, - { - name: 'city_name', - level: 'core', - type: 'keyword', - description: 'City name.', - example: 'Montreal', - }, - { - name: 'country_iso_code', - level: 'core', - type: 'keyword', - description: 'Country ISO code.', - example: 'CA', - }, - { - name: 'region_iso_code', - level: 'core', - type: 'keyword', - description: 'Region ISO code.', - example: 'CA-QC', - }, - { - name: 'name', - level: 'extended', - type: 'keyword', - description: - 'User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation.', - example: 'boston-dc', - }, - ], + name: 'region_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Region name.', + example: 'Quebec', }, ], }, { - name: 'service', - title: 'Service', + name: 'group', + title: 'Group', group: 2, description: - 'The service fields describe the service for or from which the data was collected. These fields help you find and correlate logs for a specific service and version.', + 'The group fields are meant to represent groups that are relevant\nto the event.', type: 'group', fields: [ { - name: 'id', - level: 'core', + name: 'domain', + level: 'extended', type: 'keyword', + ignore_above: 1024, description: - 'Unique identifier of the running service. This id should uniquely identify this service. This makes it possible to correlate logs and metrics for one specific service. Example: If you are experiencing issues with one redis instance, you can filter on that id to see metrics and logs for that single instance.', - example: 'd37e5ebfe0ae6c4972dbe9f0174a1637bb8247f6', + 'Name of the directory the group is a member of.\n\nFor example, an LDAP or Active Directory domain name.', + }, + { + name: 'id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifier for the group on the system/platform.', }, { name: 'name', - level: 'core', + level: 'extended', type: 'keyword', - example: 'elasticsearch-metrics', - description: - 'Name of the service data is collected from. The name of the service is normally user given. This allows if two instances of the same service are running on the same machine they can be differentiated by the `service.name`. Also it allows for distributed services that run on multiple hosts to correlate the related instances based on the name. In the case of Elasticsearch the service.name could contain the cluster name. For Beats the service.name is by default a copy of the `service.type` field if no name is specified.', + ignore_above: 1024, + description: 'Name of the group.', }, + ], + }, + { + name: 'hash', + title: 'Hash', + group: 2, + description: + 'The hash fields represent different hash algorithms and their values.\n\nField names for common hashes (e.g. MD5, SHA1) are predefined. Add fields for\nother hashes by lowercasing the hash algorithm name and using underscore separators\nas appropriate (snake case, e.g. sha3_512).', + type: 'group', + fields: [ { - name: 'type', - level: 'core', + name: 'md5', + level: 'extended', type: 'keyword', - example: 'elasticsearch', - description: - 'The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`.', + ignore_above: 1024, + description: 'MD5 hash.', }, { - name: 'state', - level: 'core', + name: 'sha1', + level: 'extended', type: 'keyword', - description: 'Current state of the service.', + ignore_above: 1024, + description: 'SHA1 hash.', }, { - name: 'version', - level: 'core', + name: 'sha256', + level: 'extended', type: 'keyword', - example: '3.2.4', - description: - 'Version of the service the data was collected from. This allows to look at a data set only for a specific version of a service.', + ignore_above: 1024, + description: 'SHA256 hash.', }, { - name: 'ephemeral_id', + name: 'sha512', level: 'extended', type: 'keyword', - description: - 'Ephemeral identifier of this service (if one exists). This id normally changes across restarts, but `service.id` does not.', - example: '8a4f500f', + ignore_above: 1024, + description: 'SHA512 hash.', }, ], }, { - name: 'source', - title: 'Source', + name: 'host', + title: 'Host', group: 2, description: - 'Source fields describe details about the source of a packet/event. Source fields are usually populated in conjunction with destination fields.', + 'A host is defined as a general computing instance.\n\nECS host.* fields should be populated with details about the host on which the\nevent happened, or from which the measurement was taken. Host types include\nhardware, virtual machines, Docker containers, and Kubernetes nodes.', type: 'group', fields: [ { - name: 'address', + name: 'architecture', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Operating system architecture.', + example: 'x86_64', + }, + { + name: 'domain', level: 'extended', type: 'keyword', + ignore_above: 1024, description: - 'Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is.', + 'Name of the domain of which the host is a member.\n\nFor example, on Windows this could be the host Active Directory domain\nor NetBIOS domain name. For Linux this could be the domain of the host\nLDAP provider.', + example: 'CONTOSO', + default_field: false, }, { - name: 'ip', + name: 'geo.city_name', level: 'core', - type: 'ip', - description: 'IP address of the source. Can be one or multiple IPv4 or IPv6 addresses.', + type: 'keyword', + ignore_above: 1024, + description: 'City name.', + example: 'Montreal', }, { - name: 'port', + name: 'geo.continent_name', level: 'core', - type: 'long', - description: 'Port of the source.', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the continent.', + example: 'North America', }, { - name: 'mac', + name: 'geo.country_iso_code', level: 'core', type: 'keyword', - description: 'MAC address of the source.', + ignore_above: 1024, + description: 'Country ISO code.', + example: 'CA', }, { - name: 'domain', + name: 'geo.country_name', level: 'core', type: 'keyword', - description: 'Source domain.', + ignore_above: 1024, + description: 'Country name.', + example: 'Canada', }, { - name: 'bytes', + name: 'geo.location', level: 'core', - type: 'long', - format: 'bytes', - example: 184, - description: 'Bytes sent from the source to the destination.', + type: 'geo_point', + description: 'Longitude and latitude.', + example: '{ "lon": -73.614830, "lat": 45.505918 }', }, { - name: 'packets', + name: 'geo.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'User-defined description of a location, at the level of granularity\nthey care about.\n\nCould be the name of their data centers, the floor number, if this describes\na local physical entity, city names.\n\nNot typically used in automated geolocation.', + example: 'boston-dc', + }, + { + name: 'geo.region_iso_code', level: 'core', - type: 'long', - example: 12, - description: 'Packets sent from the source to the destination.', + type: 'keyword', + ignore_above: 1024, + description: 'Region ISO code.', + example: 'CA-QC', }, { - name: 'geo', - title: 'Geo', - group: 2, - description: - 'Geo fields can carry data about a specific location related to an event or geo information derived from an IP field.', - type: 'group', - fields: [ - { - name: 'location', - level: 'core', - type: 'geo_point', - description: 'Longitude and latitude.', - example: '{ "lon": -73.614830, "lat": 45.505918 }', - }, - { - name: 'continent_name', - level: 'core', - type: 'keyword', - description: 'Name of the continent.', - example: 'North America', - }, - { - name: 'country_name', - level: 'core', - type: 'keyword', - description: 'Country name.', - example: 'Canada', - }, - { - name: 'region_name', - level: 'core', - type: 'keyword', - description: 'Region name.', - example: 'Quebec', - }, - { - name: 'city_name', - level: 'core', - type: 'keyword', - description: 'City name.', - example: 'Montreal', - }, - { - name: 'country_iso_code', - level: 'core', - type: 'keyword', - description: 'Country ISO code.', - example: 'CA', - }, - { - name: 'region_iso_code', - level: 'core', - type: 'keyword', - description: 'Region ISO code.', - example: 'CA-QC', - }, - { - name: 'name', - level: 'extended', - type: 'keyword', - description: - 'User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation.', - example: 'boston-dc', - }, - ], + name: 'geo.region_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Region name.', + example: 'Quebec', }, - ], - }, - { - name: 'url', - title: 'URL', - description: 'URL fields provide a complete URL, with scheme, host, and path.', - type: 'group', - fields: [ { - name: 'original', - level: 'extended', + name: 'hostname', + level: 'core', type: 'keyword', + ignore_above: 1024, description: - 'Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not.', - example: - 'https://www.elastic.co:443/search?q=elasticsearch#top or /search?q=elasticsearch', + 'Hostname of the host.\n\nIt normally contains what the `hostname` command returns on the host machine.', }, { - name: 'full', - level: 'extended', + name: 'id', + level: 'core', type: 'keyword', + ignore_above: 1024, description: - 'If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source.', - example: 'https://www.elastic.co:443/search?q=elasticsearch#top', + 'Unique host id.\n\nAs hostname is not always unique, use values that are meaningful in your environment.\n\nExample: The current usage of `beat.name`.', }, { - name: 'scheme', - level: 'extended', + name: 'ip', + level: 'core', + type: 'ip', + description: 'Host ip addresses.', + }, + { + name: 'mac', + level: 'core', type: 'keyword', - description: - 'Scheme of the request, such as "https". Note: The `:` is not part of the scheme.', - example: 'https', + ignore_above: 1024, + description: 'Host mac addresses.', }, { - name: 'domain', - level: 'extended', + name: 'name', + level: 'core', type: 'keyword', + ignore_above: 1024, description: - 'Domain of the request, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field.', - example: 'www.elastic.co', + 'Name of the host.\n\nIt can contain what `hostname` returns on Unix systems, the fully qualified\ndomain name, or a name specified by the user. The sender decides which value\nto use.', }, { - name: 'port', + name: 'os.family', level: 'extended', - type: 'integer', - description: 'Port of the request, such as 443.', - example: 443, + type: 'keyword', + ignore_above: 1024, + description: 'OS family (such as redhat, debian, freebsd, windows).', + example: 'debian', }, { - name: 'path', + name: 'os.full', level: 'extended', type: 'keyword', - description: 'Path of the request, such as "/search".', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Operating system name, including the version or code name.', + example: 'Mac OS Mojave', }, { - name: 'query', + name: 'os.kernel', level: 'extended', type: 'keyword', - description: - 'The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases.', + ignore_above: 1024, + description: 'Operating system kernel version as a raw string.', + example: '4.4.0-112-generic', }, { - name: 'fragment', + name: 'os.name', level: 'extended', type: 'keyword', - description: - 'Portion of the url after the `#`, such as "top". The `#` is not part of the fragment.', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Operating system name, without the version.', + example: 'Mac OS X', }, { - name: 'username', + name: 'os.platform', level: 'extended', type: 'keyword', - description: 'Username of the request.', + ignore_above: 1024, + description: 'Operating system platform (such centos, ubuntu, windows).', + example: 'darwin', }, { - name: 'password', + name: 'os.version', level: 'extended', type: 'keyword', - description: 'Password of the request.', + ignore_above: 1024, + description: 'Operating system version as a raw string.', + example: '10.14.1', }, - ], - }, - { - name: 'user', - title: 'User', - group: 2, - description: - 'The user fields describe information about the user that is relevant to the event. Fields can have one entry or multiple entries. If a user has more than one id, provide an array that includes all of them.', - reusable: { - top_level: true, - expected: ['client', 'destination', 'host', 'server', 'source'], - }, - type: 'group', - fields: [ { - name: 'id', + name: 'type', level: 'core', type: 'keyword', - description: 'One or multiple unique identifiers of the user.', + ignore_above: 1024, + description: + 'Type of host.\n\nFor Cloud providers this can be the machine type like `t2.medium`. If vm,\nthis could be the container, for example, or other information meaningful\nin your environment.', }, { - name: 'name', - level: 'core', - type: 'keyword', - example: 'albert', - description: 'Short name or login of the user.', + name: 'uptime', + level: 'extended', + type: 'long', + description: 'Seconds the host has been up.', + example: 1325, }, { - name: 'full_name', + name: 'user.domain', level: 'extended', type: 'keyword', - example: 'Albert Einstein', - description: "User's full name, if available. ", + ignore_above: 1024, + description: + 'Name of the directory the user is a member of.\n\nFor example, an LDAP or Active Directory domain name.', }, { - name: 'email', + name: 'user.email', level: 'extended', type: 'keyword', + ignore_above: 1024, description: 'User email address.', }, { - name: 'hash', + name: 'user.full_name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: "User's full name, if available.", + example: 'Albert Einstein', + }, + { + name: 'user.group.domain', level: 'extended', type: 'keyword', + ignore_above: 1024, description: - 'Unique user hash to correlate information for a user in anonymized form. Useful if `user.id` or `user.name` contain confidential information and cannot be used.', + 'Name of the directory the group is a member of.\n\nFor example, an LDAP or Active Directory domain name.', }, { - name: 'group', - title: 'Group', - group: 2, + name: 'user.group.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifier for the group on the system/platform.', + }, + { + name: 'user.group.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the group.', + }, + { + name: 'user.hash', + level: 'extended', + type: 'keyword', + ignore_above: 1024, description: - 'The group fields are meant to represent groups that are relevant to the event.', - type: 'group', - fields: [ - { - name: 'id', - level: 'extended', - type: 'keyword', - description: 'Unique identifier for the group on the system/platform.', - }, + 'Unique user hash to correlate information for a user in anonymized\nform.\n\nUseful if `user.id` or `user.name` contain confidential information and cannot\nbe used.', + }, + { + name: 'user.id', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifiers of the user.', + }, + { + name: 'user.name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ { - name: 'name', - level: 'extended', - type: 'keyword', - description: 'Name of the group.', + name: 'text', + type: 'text', + norms: false, + default_field: false, }, ], + description: 'Short name or login of the user.', + example: 'albert', }, ], }, { - name: 'user_agent', - title: 'User agent', + name: 'http', + title: 'HTTP', group: 2, description: - 'The user_agent fields normally come from a browser request. They often show up in web service logs coming from the parsed user agent string.', + 'Fields related to HTTP activity. Use the `url` field set to store\nthe url of the request.', type: 'group', fields: [ { - name: 'original', + name: 'request.body.bytes', level: 'extended', - type: 'keyword', - description: 'Unparsed version of the user_agent.', - example: - 'Mozilla/5.0 (iPhone; CPU iPhone OS 12_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1', + type: 'long', + format: 'bytes', + description: 'Size in bytes of the request body.', + example: 887, }, { - name: 'name', + name: 'request.body.content', level: 'extended', type: 'keyword', - example: 'Safari', - description: 'Name of the user agent.', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'The full HTTP request body.', + example: 'Hello world', }, { - name: 'version', + name: 'request.bytes', + level: 'extended', + type: 'long', + format: 'bytes', + description: 'Total size in bytes of the request (body and headers).', + example: 1437, + }, + { + name: 'request.method', level: 'extended', type: 'keyword', - description: 'Version of the user agent.', - example: 12, + ignore_above: 1024, + description: + 'HTTP request method.\n\nThe field value must be normalized to lowercase for querying. See the documentation\nsection "Implementing ECS".', + example: 'get, post, put', }, { - name: 'device.name', + name: 'request.referrer', level: 'extended', type: 'keyword', - example: 'iPhone', - description: 'Name of the device.', + ignore_above: 1024, + description: 'Referrer for this HTTP request.', + example: 'https://blog.example.com/', }, { - name: 'os', - title: 'Operating System', - group: 2, - description: 'The OS fields contain information about the operating system.', - reusable: { - top_level: false, - expected: ['observer', 'host', 'user_agent'], - }, - type: 'group', - fields: [ - { - name: 'platform', - level: 'extended', - type: 'keyword', - description: 'Operating system platform (such centos, ubuntu, windows).', - example: 'darwin', - }, - { - name: 'name', - level: 'extended', - type: 'keyword', - example: 'Mac OS X', - description: 'Operating system name, without the version.', - }, - { - name: 'full', - level: 'extended', - type: 'keyword', - example: 'Mac OS Mojave', - description: 'Operating system name, including the version or code name.', - }, - { - name: 'family', - level: 'extended', - type: 'keyword', - example: 'debian', - description: 'OS family (such as redhat, debian, freebsd, windows).', - }, - { - name: 'version', - level: 'extended', - type: 'keyword', - example: '10.14.1', - description: 'Operating system version as a raw string.', - }, + name: 'response.body.bytes', + level: 'extended', + type: 'long', + format: 'bytes', + description: 'Size in bytes of the response body.', + example: 887, + }, + { + name: 'response.body.content', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ { - name: 'kernel', - level: 'extended', - type: 'keyword', - example: '4.4.0-112-generic', - description: 'Operating system kernel version as a raw string.', + name: 'text', + type: 'text', + norms: false, + default_field: false, }, ], + description: 'The full HTTP response body.', + example: 'Hello world', + }, + { + name: 'response.bytes', + level: 'extended', + type: 'long', + format: 'bytes', + description: 'Total size in bytes of the response (body and headers).', + example: 1437, + }, + { + name: 'response.status_code', + level: 'extended', + type: 'long', + format: 'string', + description: 'HTTP response status code.', + example: 404, + }, + { + name: 'version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'HTTP version.', + example: 1.1, }, ], }, { - name: 'agent.hostname', - type: 'keyword', - description: 'Hostname of the agent.', - }, - ], - }, - { - key: 'beat', - title: 'Beat', - description: 'Contains common beat fields available in all event types.', - fields: [ - { - name: 'beat.timezone', - type: 'alias', - path: 'event.timezone', - migration: true, - }, - { - name: 'fields', - type: 'object', - object_type: 'keyword', - description: 'Contains user configurable fields.', - }, - { - name: 'error', + name: 'interface', + title: 'Interface', + group: 2, + description: + 'The interface fields are used to record ingress and egress interface\ninformation when reported by an observer (e.g. firewall, router, load balancer)\nin the context of the observer handling a network connection. In the case of\na single observer interface (e.g. network sensor on a span port) only the observer.ingress\ninformation should be populated.', type: 'group', - description: 'Error fields containing additional info in case of errors.', fields: [ { - name: 'type', + name: 'alias', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Interface alias as reported by the system, typically used in firewall\nimplementations for e.g. inside, outside, or dmz logical interface naming.', + example: 'outside', + default_field: false, + }, + { + name: 'id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Interface ID as reported by an observer (typically SNMP interface\nID).', + example: 10, + default_field: false, + }, + { + name: 'name', + level: 'extended', type: 'keyword', - description: 'Error type.', + ignore_above: 1024, + description: 'Interface name as reported by the system.', + example: 'eth0', + default_field: false, }, ], }, { - name: 'beat.name', - type: 'alias', - path: 'host.name', - migration: true, - }, - { - name: 'beat.hostname', - type: 'alias', - path: 'agent.hostname', - migration: true, - }, - ], - }, - { - key: 'cloud', - title: 'Cloud provider metadata', - description: 'Metadata from cloud providers added by the add_cloud_metadata processor.', - fields: [ - { - name: 'cloud.project.id', - example: 'project-x', - description: 'Name of the project in Google Cloud.', - }, - { - name: 'meta.cloud.provider', - type: 'alias', - path: 'cloud.provider', - migration: true, - }, - { - name: 'meta.cloud.instance_id', - type: 'alias', - path: 'cloud.instance.id', - migration: true, - }, - { - name: 'meta.cloud.instance_name', - type: 'alias', - path: 'cloud.instance.name', - migration: true, - }, - { - name: 'meta.cloud.machine_type', - type: 'alias', - path: 'cloud.machine.type', - migration: true, - }, - { - name: 'meta.cloud.availability_zone', - type: 'alias', - path: 'cloud.availability_zone', - migration: true, - }, - { - name: 'meta.cloud.project_id', - type: 'alias', - path: 'cloud.project.id', - migration: true, - }, - { - name: 'meta.cloud.region', - type: 'alias', - path: 'cloud.region', - migration: true, - }, - ], - }, - { - key: 'docker', - title: 'Docker', - description: 'Docker stats collected from Docker.', - short_config: false, - anchor: 'docker-processor', - fields: [ - { - name: 'docker', + name: 'log', + title: 'Log', + group: 2, + description: + 'Details about the event logging mechanism or logging transport.\n\nThe log.* fields are typically populated with details about the logging mechanism\nused to create and/or transport the event. For example, syslog details belong\nunder `log.syslog.*`.\n\nThe details specific to your event source are typically not logged under `log.*`,\nbut rather in `event.*` or in other ECS fields.', type: 'group', fields: [ { - name: 'container.id', - type: 'alias', - path: 'container.id', - migration: true, + name: 'level', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'Original log level of the log event.\n\nIf the source of the event provides a log level or textual severity, this\nis the one that goes in `log.level`. If your source does not specify one,\nyou may put your event transport severity here (e.g. Syslog severity).\n\nSome examples are `warn`, `err`, `i`, `informational`.', + example: 'error', }, { - name: 'container.image', - type: 'alias', - path: 'container.image.name', - migration: true, + name: 'logger', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'The name of the logger inside an application. This is usually the\nname of the class which initialized the logger, or can be a custom name.', + example: 'org.elasticsearch.bootstrap.Bootstrap', }, { - name: 'container.name', - type: 'alias', - path: 'container.name', - migration: true, + name: 'origin.file.line', + level: 'extended', + type: 'integer', + description: + 'The line number of the file containing the source code which originated\nthe log event.', + example: 42, }, { - name: 'container.labels', + name: 'origin.file.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The name of the file containing the source code which originated\nthe log event. Note that this is not the name of the log file.', + example: 'Bootstrap.java', + }, + { + name: 'origin.function', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'The name of the function or method which originated the log event.', + example: 'init', + }, + { + name: 'original', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'This is the original log message and contains the full log message\nbefore splitting it up in multiple parts.\n\nIn contrast to the `message` field which can contain an extracted part of\nthe log message, this field contains the original, full log message. It can\nhave already some modifications applied like encoding or new lines removed\nto clean up the log message.\n\nThis field is not indexed and doc_values are disabled so it cannot be queried\nbut the value can be retrieved from `_source`.', + example: 'Sep 19 08:26:10 localhost My log', + }, + { + name: 'syslog', + level: 'extended', type: 'object', object_type: 'keyword', - description: 'Image labels.', + description: + 'The Syslog metadata of the event, if the event was transmitted\nvia Syslog. Please see RFCs 5424 or 3164.', + }, + { + name: 'syslog.facility.code', + level: 'extended', + type: 'long', + format: 'string', + description: + 'The Syslog numeric facility of the log event, if available.\n\nAccording to RFCs 5424 and 3164, this value should be an integer between 0\nand 23.', + example: 23, + }, + { + name: 'syslog.facility.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'The Syslog text-based facility of the log event, if available.', + example: 'local7', + }, + { + name: 'syslog.priority', + level: 'extended', + type: 'long', + format: 'string', + description: + 'Syslog numeric priority of the event, if available.\n\nAccording to RFCs 5424 and 3164, the priority is 8 * facility + severity.\nThis number is therefore expected to contain a value between 0 and 191.', + example: 135, + }, + { + name: 'syslog.severity.code', + level: 'extended', + type: 'long', + description: + 'The Syslog numeric severity of the log event, if available.\n\nIf the event source publishing via Syslog provides a different numeric severity\nvalue (e.g. firewall, IDS), your source numeric severity should go to `event.severity`.\nIf the event source does not specify a distinct severity, you can optionally\ncopy the Syslog severity to `event.severity`.', + example: 3, + }, + { + name: 'syslog.severity.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The Syslog numeric severity of the log event, if available.\n\nIf the event source publishing via Syslog provides a different severity value\n(e.g. firewall, IDS), your source text severity should go to `log.level`.\nIf the event source does not specify a distinct severity, you can optionally\ncopy the Syslog severity to `log.level`.', + example: 'Error', }, ], }, - ], - }, - { - key: 'host', - title: 'Host', - description: 'Info collected for the host machine.', - anchor: 'host-processor', - }, - { - key: 'kubernetes', - title: 'Kubernetes', - description: 'Kubernetes metadata added by the kubernetes processor', - short_config: false, - anchor: 'kubernetes-processor', - fields: [ { - name: 'kubernetes', + name: 'network', + title: 'Network', + group: 2, + description: + 'The network is defined as the communication path over which a host\nor network event happens.\n\nThe network.* fields should be populated with details about the network activity\nassociated with an event.', type: 'group', fields: [ { - name: 'pod.name', + name: 'application', + level: 'extended', type: 'keyword', - description: 'Kubernetes pod name', + ignore_above: 1024, + description: + 'A name given to an application level protocol. This can be arbitrarily\nassigned for things like microservices, but also apply to things like skype,\nicq, facebook, twitter. This would be used in situations where the vendor\nor service can be decoded such as from the source/dest IP owners, ports, or\nwire format.\n\nThe field value must be normalized to lowercase for querying. See the documentation\nsection "Implementing ECS".', + example: 'aim', }, { - name: 'pod.uid', + name: 'bytes', + level: 'core', + type: 'long', + format: 'bytes', + description: + 'Total bytes transferred in both directions.\n\nIf `source.bytes` and `destination.bytes` are known, `network.bytes` is their\nsum.', + example: 368, + }, + { + name: 'community_id', + level: 'extended', type: 'keyword', - description: 'Kubernetes Pod UID', + ignore_above: 1024, + description: + 'A hash of source and destination IPs and ports, as well as the\nprotocol used in a communication. This is a tool-agnostic standard to identify\nflows.\n\nLearn more at https://github.com/corelight/community-id-spec.', + example: '1:hO+sN4H+MG5MY/8hIrXPqc4ZQz0=', }, { - name: 'namespace', + name: 'direction', + level: 'core', type: 'keyword', - description: 'Kubernetes namespace', + ignore_above: 1024, + description: + "Direction of the network traffic.\nRecommended values are:\n * inbound\n * outbound\n * internal\n * external\n * unknown\n\nWhen mapping events from a host-based monitoring context, populate this field from the host's point of view.\nWhen mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of your network perimeter.", + example: 'inbound', }, { - name: 'node.name', + name: 'forwarded_ip', + level: 'core', + type: 'ip', + description: 'Host IP address when the source IP address is the proxy.', + example: '192.1.1.2', + }, + { + name: 'iana_number', + level: 'extended', type: 'keyword', - description: 'Kubernetes node name', + ignore_above: 1024, + description: + 'IANA Protocol Number (https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml).\nStandardized list of protocols. This aligns well with NetFlow and sFlow related\nlogs which use the IANA Protocol Number.', + example: 6, }, { - name: 'labels', + name: 'inner', + level: 'extended', type: 'object', - description: 'Kubernetes labels map', + object_type: 'keyword', + description: + 'Network.inner fields are added in addition to network.vlan fields\nto describe the innermost VLAN when q-in-q VLAN tagging is present. Allowed\nfields include vlan.id and vlan.name. Inner vlan fields are typically used\nwhen sending traffic with multiple 802.1q encapsulations to a network sensor\n(e.g. Zeek, Wireshark.)', + default_field: false, }, { - name: 'annotations', - type: 'object', - description: 'Kubernetes annotations map', + name: 'inner.vlan.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'VLAN ID as reported by the observer.', + example: 10, + default_field: false, }, { - name: 'container.name', + name: 'inner.vlan.name', + level: 'extended', type: 'keyword', - description: 'Kubernetes container name', + ignore_above: 1024, + description: 'Optional VLAN name as reported by the observer.', + example: 'outside', + default_field: false, }, { - name: 'container.image', + name: 'name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Name given by operators to sections of their network.', + example: 'Guest Wifi', + }, + { + name: 'packets', + level: 'core', + type: 'long', + description: + 'Total packets transferred in both directions.\n\nIf `source.packets` and `destination.packets` are known, `network.packets`\nis their sum.', + example: 24, + }, + { + name: 'protocol', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'L7 Network protocol name. ex. http, lumberjack, transport protocol.\n\nThe field value must be normalized to lowercase for querying. See the documentation\nsection "Implementing ECS".', + example: 'http', + }, + { + name: 'transport', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'Same as network.iana_number, but instead using the Keyword name\nof the transport layer (udp, tcp, ipv6-icmp, etc.)\n\nThe field value must be normalized to lowercase for querying. See the documentation\nsection "Implementing ECS".', + example: 'tcp', + }, + { + name: 'type', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'In the OSI Model this would be the Network Layer. ipv4, ipv6,\nipsec, pim, etc\n\nThe field value must be normalized to lowercase for querying. See the documentation\nsection "Implementing ECS".', + example: 'ipv4', + }, + { + name: 'vlan.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'VLAN ID as reported by the observer.', + example: 10, + default_field: false, + }, + { + name: 'vlan.name', + level: 'extended', type: 'keyword', - description: 'Kubernetes container image', + ignore_above: 1024, + description: 'Optional VLAN name as reported by the observer.', + example: 'outside', + default_field: false, }, ], }, - ], - }, - { - key: 'process', - title: 'Process', - description: 'Process metadata fields', - fields: [ { - name: 'process', + name: 'observer', + title: 'Observer', + group: 2, + description: + 'An observer is defined as a special network, security, or application\ndevice used to detect, observe, or create network, security, or application-related\nevents and metrics.\n\nThis could be a custom hardware appliance or a server that has been configured\nto run special network, security, or application software. Examples include\nfirewalls, web proxies, intrusion detection/prevention systems, network monitoring\nsensors, web application firewalls, data loss prevention systems, and APM servers.\nThe observer.* fields shall be populated with details of the system, if any,\nthat detects, observes and/or creates a network, security, or application event\nor metric. Message queues and ETL components used in processing events or metrics\nare not considered observers in ECS.', type: 'group', fields: [ { - name: 'exe', - type: 'alias', - path: 'process.executable', - migration: true, + name: 'egress', + level: 'extended', + type: 'object', + object_type: 'keyword', + description: + 'Observer.egress holds information like interface number and name,\nvlan, and zone information to classify egress traffic. Single armed monitoring\nsuch as a network sensor on a span port should only use observer.ingress\nto categorize traffic.', + default_field: false, }, - ], - }, - ], - }, - { - key: 'log', - title: 'Log file content', - description: 'Contains log file lines.', - fields: [ - { - name: 'log.file.path', - type: 'keyword', - required: false, - description: - 'The file from which the line was read. This field contains the absolute path to the file. For example: `/var/log/system.log`.', - }, - { - name: 'log.source.address', - type: 'keyword', - required: false, - description: 'Source address from which the log event was read / sent from.', - }, - { - name: 'log.offset', - type: 'long', - required: false, - description: 'The file offset the reported line starts at.', - }, - { - name: 'stream', - type: 'keyword', - required: false, - description: "Log stream when reading container logs, can be 'stdout' or 'stderr' ", - }, - { - name: 'input.type', - required: true, - description: - 'The input type from which the event was generated. This field is set to the value specified for the `type` option in the input section of the Filebeat config file.', - }, - { - name: 'event.sequence', - type: 'long', - required: false, - description: 'The sequence number of this event.', - }, - { - name: 'syslog.facility', - type: 'long', - required: false, - description: 'The facility extracted from the priority.', - }, - { - name: 'syslog.priority', - type: 'long', - required: false, - description: 'The priority of the syslog event.', - }, - { - name: 'syslog.severity_label', - type: 'keyword', - required: false, - description: 'The human readable severity.', - }, - { - name: 'syslog.facility_label', - type: 'keyword', - required: false, - description: 'The human readable facility.', - }, - { - name: 'process.program', - type: 'keyword', - required: false, - description: 'The name of the program.', - }, - { - name: 'log.flags', - description: 'This field contains the flags of the event.', - }, - { - name: 'http.response.content_length', - type: 'alias', - path: 'http.response.body.bytes', - migration: true, - }, - { - name: 'user_agent', - type: 'group', - fields: [ { - name: 'os', - type: 'group', - fields: [ + name: 'egress.interface.alias', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Interface alias as reported by the system, typically used in firewall\nimplementations for e.g. inside, outside, or dmz logical interface naming.', + example: 'outside', + default_field: false, + }, + { + name: 'egress.interface.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Interface ID as reported by an observer (typically SNMP interface\nID).', + example: 10, + default_field: false, + }, + { + name: 'egress.interface.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Interface name as reported by the system.', + example: 'eth0', + default_field: false, + }, + { + name: 'egress.vlan.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'VLAN ID as reported by the observer.', + example: 10, + default_field: false, + }, + { + name: 'egress.vlan.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Optional VLAN name as reported by the observer.', + example: 'outside', + default_field: false, + }, + { + name: 'egress.zone', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Network zone of outbound traffic as reported by the observer to\ncategorize the destination area of egress traffic, e.g. Internal, External,\nDMZ, HR, Legal, etc.', + example: 'Public_Internet', + default_field: false, + }, + { + name: 'geo.city_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'City name.', + example: 'Montreal', + }, + { + name: 'geo.continent_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the continent.', + example: 'North America', + }, + { + name: 'geo.country_iso_code', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Country ISO code.', + example: 'CA', + }, + { + name: 'geo.country_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Country name.', + example: 'Canada', + }, + { + name: 'geo.location', + level: 'core', + type: 'geo_point', + description: 'Longitude and latitude.', + example: '{ "lon": -73.614830, "lat": 45.505918 }', + }, + { + name: 'geo.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'User-defined description of a location, at the level of granularity\nthey care about.\n\nCould be the name of their data centers, the floor number, if this describes\na local physical entity, city names.\n\nNot typically used in automated geolocation.', + example: 'boston-dc', + }, + { + name: 'geo.region_iso_code', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Region ISO code.', + example: 'CA-QC', + }, + { + name: 'geo.region_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Region name.', + example: 'Quebec', + }, + { + name: 'hostname', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Hostname of the observer.', + }, + { + name: 'ingress', + level: 'extended', + type: 'object', + object_type: 'keyword', + description: + 'Observer.ingress holds information like interface number and name,\nvlan, and zone information to classify ingress traffic. Single armed monitoring\nsuch as a network sensor on a span port should only use observer.ingress\nto categorize traffic.', + default_field: false, + }, + { + name: 'ingress.interface.alias', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Interface alias as reported by the system, typically used in firewall\nimplementations for e.g. inside, outside, or dmz logical interface naming.', + example: 'outside', + default_field: false, + }, + { + name: 'ingress.interface.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Interface ID as reported by an observer (typically SNMP interface\nID).', + example: 10, + default_field: false, + }, + { + name: 'ingress.interface.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Interface name as reported by the system.', + example: 'eth0', + default_field: false, + }, + { + name: 'ingress.vlan.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'VLAN ID as reported by the observer.', + example: 10, + default_field: false, + }, + { + name: 'ingress.vlan.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Optional VLAN name as reported by the observer.', + example: 'outside', + default_field: false, + }, + { + name: 'ingress.zone', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Network zone of incoming traffic as reported by the observer to\ncategorize the source area of ingress traffic. e.g. internal, External, DMZ,\nHR, Legal, etc.', + example: 'DMZ', + default_field: false, + }, + { + name: 'ip', + level: 'core', + type: 'ip', + description: 'IP addresses of the observer.', + }, + { + name: 'mac', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'MAC addresses of the observer', + }, + { + name: 'name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Custom name of the observer.\n\nThis is a name that can be given to an observer. This can be helpful for example\nif multiple firewalls of the same model are used in an organization.\n\nIf no custom name is needed, the field can be left empty.', + example: '1_proxySG', + }, + { + name: 'os.family', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'OS family (such as redhat, debian, freebsd, windows).', + example: 'debian', + }, + { + name: 'os.full', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ { - name: 'full_name', - type: 'keyword', + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Operating system name, including the version or code name.', + example: 'Mac OS Mojave', + }, + { + name: 'os.kernel', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Operating system kernel version as a raw string.', + example: '4.4.0-112-generic', + }, + { + name: 'os.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, }, ], + description: 'Operating system name, without the version.', + example: 'Mac OS X', + }, + { + name: 'os.platform', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Operating system platform (such centos, ubuntu, windows).', + example: 'darwin', + }, + { + name: 'os.version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Operating system version as a raw string.', + example: '10.14.1', + }, + { + name: 'product', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'The product name of the observer.', + example: 's200', + }, + { + name: 'serial_number', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Observer serial number.', + }, + { + name: 'type', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'The type of the observer the data is coming from.\n\nThere is no predefined list of observer types. Some examples are `forwarder`,\n`firewall`, `ids`, `ips`, `proxy`, `poller`, `sensor`, `APM server`.', + example: 'firewall', + }, + { + name: 'vendor', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Vendor name of the observer.', + example: 'Symantec', + }, + { + name: 'version', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Observer version.', }, ], }, { - name: 'fileset.name', - type: 'keyword', - description: 'The Filebeat fileset that generated this event.', - }, - { - name: 'fileset.module', - type: 'alias', - path: 'event.module', - migration: true, - }, - { - name: 'read_timestamp', - type: 'alias', - path: 'event.created', - migration: true, - }, - ], - }, - { - key: 'apache', - title: 'Apache', - description: 'Apache Module', - short_config: true, - fields: [ - { - name: 'apache2', + name: 'organization', + title: 'Organization', + group: 2, + description: + 'The organization fields enrich data with information about the company\nor entity the data is associated with.\n\nThese fields help you arrange or filter data stored in an index by one or multiple\norganizations.', type: 'group', - description: 'Aliases for backward compatibility with old apache2 fields', fields: [ { - name: 'access', - type: 'group', - fields: [ + name: 'id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifier for the organization.', + }, + { + name: 'name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ { - name: 'remote_ip', - type: 'alias', - path: 'source.address', - migration: true, - }, - { - name: 'ssl.protocol', - type: 'alias', - path: 'apache.access.ssl.protocol', - migration: true, - }, - { - name: 'ssl.cipher', - type: 'alias', - path: 'apache.access.ssl.cipher', - migration: true, - }, - { - name: 'body_sent.bytes', - type: 'alias', - path: 'http.response.body.bytes', - migration: true, - }, - { - name: 'user_name', - type: 'alias', - path: 'user.name', - migration: true, - }, - { - name: 'method', - type: 'alias', - path: 'http.request.method', - migration: true, - }, - { - name: 'url', - type: 'alias', - path: 'url.original', - migration: true, - }, - { - name: 'http_version', - type: 'alias', - path: 'http.version', - migration: true, - }, - { - name: 'response_code', - type: 'alias', - path: 'http.response.status_code', - migration: true, - }, - { - name: 'referrer', - type: 'alias', - path: 'http.request.referrer', - migration: true, - }, - { - name: 'agent', - type: 'alias', - path: 'user_agent.original', - migration: true, - }, - { - name: 'user_agent', - type: 'group', - fields: [ - { - name: 'device', - type: 'alias', - path: 'user_agent.device.name', - migration: true, - }, - { - name: 'name', - type: 'alias', - path: 'user_agent.name', - migration: true, - }, - { - name: 'os', - type: 'alias', - path: 'user_agent.os.full_name', - migration: true, - }, - { - name: 'os_name', - type: 'alias', - path: 'user_agent.os.name', - migration: true, - }, - { - name: 'original', - type: 'alias', - path: 'user_agent.original', - migration: true, - }, - ], - }, - { - name: 'geoip', - type: 'group', - fields: [ - { - name: 'continent_name', - type: 'alias', - path: 'source.geo.continent_name', - migration: true, - }, - { - name: 'country_iso_code', - type: 'alias', - path: 'source.geo.country_iso_code', - migration: true, - }, - { - name: 'location', - type: 'alias', - path: 'source.geo.location', - migration: true, - }, - { - name: 'region_name', - type: 'alias', - path: 'source.geo.region_name', - migration: true, - }, - { - name: 'city_name', - type: 'alias', - path: 'source.geo.city_name', - migration: true, - }, - { - name: 'region_iso_code', - type: 'alias', - path: 'source.geo.region_iso_code', - migration: true, - }, - ], - }, - ], - }, - { - name: 'error', - type: 'group', - fields: [ - { - name: 'level', - type: 'alias', - path: 'log.level', - migration: true, - }, - { - name: 'message', - type: 'alias', - path: 'message', - migration: true, - }, - { - name: 'pid', - type: 'alias', - path: 'process.pid', - migration: true, - }, - { - name: 'tid', - type: 'alias', - path: 'process.thread.id', - migration: true, - }, - { - name: 'module', - type: 'alias', - path: 'apache.error.module', - migration: true, + name: 'text', + type: 'text', + norms: false, + default_field: false, }, ], + description: 'Organization name.', }, ], }, { - name: 'apache', + name: 'os', + title: 'Operating System', + group: 2, + description: 'The OS fields contain information about the operating system.', type: 'group', - description: 'Apache fields.', fields: [ { - name: 'access', - type: 'group', - description: 'Contains fields for the Apache HTTP Server access logs.', - fields: [ - { - name: 'ssl.protocol', - type: 'keyword', - description: 'SSL protocol version.', - }, + name: 'family', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'OS family (such as redhat, debian, freebsd, windows).', + example: 'debian', + }, + { + name: 'full', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ { - name: 'ssl.cipher', - type: 'keyword', - description: 'SSL cipher name.', + name: 'text', + type: 'text', + norms: false, + default_field: false, }, ], + description: 'Operating system name, including the version or code name.', + example: 'Mac OS Mojave', }, { - name: 'error', - type: 'group', - description: 'Fields from the Apache error logs.', - fields: [ + name: 'kernel', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Operating system kernel version as a raw string.', + example: '4.4.0-112-generic', + }, + { + name: 'name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ { - name: 'module', - type: 'keyword', - description: 'The module producing the logged message.', + name: 'text', + type: 'text', + norms: false, + default_field: false, }, ], + description: 'Operating system name, without the version.', + example: 'Mac OS X', + }, + { + name: 'platform', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Operating system platform (such centos, ubuntu, windows).', + example: 'darwin', + }, + { + name: 'version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Operating system version as a raw string.', + example: '10.14.1', }, ], }, - ], - }, - { - key: 'auditd', - title: 'Auditd', - description: 'Module for parsing auditd logs.', - short_config: true, - fields: [ { - name: 'user', + name: 'package', + title: 'Package', + group: 2, + description: + 'These fields contain information about an installed software package.\nIt contains general information about a package, such as name, version or size.\nIt also contains installation details, such as time or location.', type: 'group', fields: [ { - name: 'terminal', + name: 'architecture', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Package architecture.', + example: 'x86_64', + }, + { + name: 'build_version', + level: 'extended', type: 'keyword', + ignore_above: 1024, description: - 'Terminal or tty device on which the user is performing the observed activity.', + 'Additional information about the build version of the installed\npackage.\n\nFor example use the commit SHA of a non-released package.', + example: '36f4f7e89dd61b0988b12ee000b98966867710cd', + default_field: false, }, { - name: 'audit', - type: 'group', - fields: [ - { - name: 'id', - type: 'keyword', - description: 'One or multiple unique identifiers of the user.', - }, - { - name: 'name', - type: 'keyword', - example: 'albert', - description: 'Short name or login of the user.', - }, - { - name: 'group.id', - type: 'keyword', - description: 'Unique identifier for the group on the system/platform.', - }, - { - name: 'group.name', - type: 'keyword', - description: 'Name of the group.', - }, - ], + name: 'checksum', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Checksum of the installed package for verification.', + example: '68b329da9893e34099c7d8ad5cb9c940', }, { - name: 'effective', - type: 'group', - fields: [ - { - name: 'id', - type: 'keyword', - description: 'One or multiple unique identifiers of the user.', - }, - { - name: 'name', - type: 'keyword', - example: 'albert', - description: 'Short name or login of the user.', - }, - { - name: 'group.id', - type: 'keyword', - description: 'Unique identifier for the group on the system/platform.', - }, - { - name: 'group.name', - type: 'keyword', - description: 'Name of the group.', - }, - ], + name: 'description', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Description of the package.', + example: + 'Open source programming language to build simple/reliable/efficient\nsoftware.', }, { - name: 'filesystem', - type: 'group', - fields: [ - { - name: 'id', - type: 'keyword', - description: 'One or multiple unique identifiers of the user.', - }, - { - name: 'name', - type: 'keyword', - example: 'albert', - description: 'Short name or login of the user.', - }, - { - name: 'group.id', - type: 'keyword', - description: 'Unique identifier for the group on the system/platform.', - }, - { - name: 'group.name', - type: 'keyword', - description: 'Name of the group.', - }, - ], + name: 'install_scope', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Indicating how the package was installed, e.g. user-local, global.', + example: 'global', }, { - name: 'owner', - type: 'group', - fields: [ - { - name: 'id', - type: 'keyword', - description: 'One or multiple unique identifiers of the user.', - }, - { - name: 'name', - type: 'keyword', - example: 'albert', - description: 'Short name or login of the user.', - }, - { - name: 'group.id', - type: 'keyword', - description: 'Unique identifier for the group on the system/platform.', - }, - { - name: 'group.name', - type: 'keyword', - description: 'Name of the group.', - }, - ], + name: 'installed', + level: 'extended', + type: 'date', + description: 'Time when package was installed.', }, { - name: 'saved', - type: 'group', - fields: [ - { - name: 'id', - type: 'keyword', - description: 'One or multiple unique identifiers of the user.', - }, - { - name: 'name', - type: 'keyword', - example: 'albert', - description: 'Short name or login of the user.', - }, - { - name: 'group.id', - type: 'keyword', - description: 'Unique identifier for the group on the system/platform.', - }, - { - name: 'group.name', - type: 'keyword', - description: 'Name of the group.', - }, - ], + name: 'license', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'License under which the package was released.\n\nUse a short name, e.g. the license identifier from SPDX License List where\npossible (https://spdx.org/licenses/).', + example: 'Apache License 2.0', + }, + { + name: 'name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Package name', + example: 'go', + }, + { + name: 'path', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Path where the package is installed.', + example: '/usr/local/Cellar/go/1.12.9/', + }, + { + name: 'reference', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Home page or reference URL of the software in this package, if\navailable.', + example: 'https://golang.org', + default_field: false, + }, + { + name: 'size', + level: 'extended', + type: 'long', + format: 'string', + description: 'Package size in bytes.', + example: 62231, + }, + { + name: 'type', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Type of package.\n\nThis should contain the package file type, rather than the package manager\nname. Examples: rpm, dpkg, brew, npm, gem, nupkg, jar.', + example: 'rpm', + default_field: false, + }, + { + name: 'version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Package version', + example: '1.12.9', }, ], }, { - name: 'auditd', + name: 'pe', + title: 'PE Header', + group: 2, + description: 'These fields contain Windows Portable Executable (PE) metadata.', type: 'group', - description: 'Fields from the auditd logs.', fields: [ { - name: 'log', - type: 'group', + name: 'company', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal company name of the file, provided at compile-time.', + example: 'Microsoft Corporation', + default_field: false, + }, + { + name: 'description', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal description of the file, provided at compile-time.', + example: 'Paint', + default_field: false, + }, + { + name: 'file_version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal version of the file, provided at compile-time.', + example: '6.3.9600.17415', + default_field: false, + }, + { + name: 'original_file_name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal name of the file, provided at compile-time.', + example: 'MSPAINT.EXE', + default_field: false, + }, + { + name: 'product', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal product name of the file, provided at compile-time.', + example: 'Microsoft® Windows® Operating System', + default_field: false, + }, + ], + }, + { + name: 'process', + title: 'Process', + group: 2, + description: + 'These fields contain information about a process.\n\nThese fields can help you correlate metrics information with a process id/name\nfrom a log message. The `process.pid` often stays in the metric itself and\nis copied to the global field for correlation.', + type: 'group', + fields: [ + { + name: 'args', + level: 'extended', + type: 'keyword', + ignore_above: 1024, description: - 'Fields from the Linux audit log. Not all fields are documented here because they are dynamic and vary by audit event type.', - fields: [ - { - name: 'old_auid', - description: - 'For login events this is the old audit ID used for the user prior to this login.', - }, - { - name: 'new_auid', - description: - 'For login events this is the new audit ID. The audit ID can be used to trace future events to the user even if their identity changes (like becoming root).', - }, - { - name: 'old_ses', - description: - 'For login events this is the old session ID used for the user prior to this login.', - }, - { - name: 'new_ses', - description: - 'For login events this is the new session ID. It can be used to tie a user to future events by session ID.', - }, - { - name: 'sequence', - type: 'long', - description: 'The audit event sequence number.', - }, - { - name: 'items', - description: 'The number of items in an event.', - }, + 'Array of process arguments, starting with the absolute path to\nthe executable.\n\nMay be filtered to protect sensitive information.', + example: ['/usr/bin/ssh', '-l', 'user', '10.0.0.16'], + }, + { + name: 'args_count', + level: 'extended', + type: 'long', + description: + 'Length of the process.args array.\n\nThis field can be useful for querying or performing bucket analysis on how\nmany arguments were provided to start a process. More arguments may be an\nindication of suspicious activity.', + example: 4, + default_field: false, + }, + { + name: 'code_signature.exists', + level: 'core', + type: 'boolean', + description: 'Boolean to capture if a signature is present.', + example: 'true', + default_field: false, + }, + { + name: 'code_signature.status', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Additional information about the certificate status.\n\nThis is useful for logging cryptographic errors with the certificate validity\nor trust status. Leave unpopulated if the validity or trust of the certificate\nwas unchecked.', + example: 'ERROR_UNTRUSTED_ROOT', + default_field: false, + }, + { + name: 'code_signature.subject_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Subject name of the code signer', + example: 'Microsoft Corporation', + default_field: false, + }, + { + name: 'code_signature.trusted', + level: 'extended', + type: 'boolean', + description: + 'Stores the trust status of the certificate chain.\n\nValidating the trust of the certificate chain may be complicated, and this\nfield should only be populated by tools that actively check the status.', + example: 'true', + default_field: false, + }, + { + name: 'code_signature.valid', + level: 'extended', + type: 'boolean', + description: + 'Boolean to capture if the digital signature is verified against\nthe binary content.\n\nLeave unpopulated if a certificate was unchecked.', + example: 'true', + default_field: false, + }, + { + name: 'command_line', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ { - name: 'item', - description: - 'The item field indicates which item out of the total number of items. This number is zero-based; a value of 0 means it is the first item.', + name: 'text', + type: 'text', + norms: false, }, + ], + description: + 'Full command line that started the process, including the absolute\npath to the executable, and all arguments.\n\nSome arguments may be filtered to protect sensitive information.', + example: '/usr/bin/ssh -l user 10.0.0.16', + default_field: false, + }, + { + name: 'entity_id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Unique identifier for the process.\n\nThe implementation of this is specified by the data source, but some examples\nof what could be used here are a process-generated UUID, Sysmon Process GUIDs,\nor a hash of some uniquely identifying components of a process.\n\nConstructing a globally unique identifier is a common practice to mitigate\nPID reuse as well as to identify a specific process over time, across multiple\nmonitored hosts.', + example: 'c2c455d9f99375d', + default_field: false, + }, + { + name: 'executable', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ { - name: 'tty', - type: 'keyword', - definition: 'TTY udevice the user is running programs on.', - }, - { - name: 'a0', - description: 'The first argument to the system call.', - }, - { - name: 'addr', - type: 'ip', - definition: 'Remote address that the user is connecting from.', - }, - { - name: 'rport', - type: 'long', - definition: 'Remote port number.', - }, - { - name: 'laddr', - type: 'ip', - definition: 'Local network address.', - }, - { - name: 'lport', - type: 'long', - definition: 'Local port number.', - }, - { - name: 'acct', - type: 'alias', - path: 'user.name', - migration: true, - }, - { - name: 'pid', - type: 'alias', - path: 'process.pid', - migration: true, - }, - { - name: 'ppid', - type: 'alias', - path: 'process.ppid', - migration: true, - }, - { - name: 'res', - type: 'alias', - path: 'event.outcome', - migration: true, - }, - { - name: 'record_type', - type: 'alias', - path: 'event.action', - migration: true, - }, - { - name: 'geoip', - type: 'group', - fields: [ - { - name: 'continent_name', - type: 'alias', - path: 'source.geo.continent_name', - migration: true, - }, - { - name: 'country_iso_code', - type: 'alias', - path: 'source.geo.country_iso_code', - migration: true, - }, - { - name: 'location', - type: 'alias', - path: 'source.geo.location', - migration: true, - }, - { - name: 'region_name', - type: 'alias', - path: 'source.geo.region_name', - migration: true, - }, - { - name: 'city_name', - type: 'alias', - path: 'source.geo.city_name', - migration: true, - }, - { - name: 'region_iso_code', - type: 'alias', - path: 'source.geo.region_iso_code', - migration: true, - }, - ], - }, - { - name: 'arch', - type: 'alias', - path: 'host.architecture', - migration: true, - }, - { - name: 'gid', - type: 'alias', - path: 'user.group.id', - migration: true, - }, - { - name: 'uid', - type: 'alias', - path: 'user.id', - migration: true, - }, - { - name: 'agid', - type: 'alias', - path: 'user.audit.group.id', - migration: true, - }, - { - name: 'auid', - type: 'alias', - path: 'user.audit.id', - migration: true, - }, - { - name: 'fsgid', - type: 'alias', - path: 'user.filesystem.group.id', - migration: true, - }, - { - name: 'fsuid', - type: 'alias', - path: 'user.filesystem.id', - migration: true, - }, - { - name: 'egid', - type: 'alias', - path: 'user.effective.group.id', - migration: true, - }, - { - name: 'euid', - type: 'alias', - path: 'user.effective.id', - migration: true, - }, - { - name: 'sgid', - type: 'alias', - path: 'user.saved.group.id', - migration: true, - }, - { - name: 'suid', - type: 'alias', - path: 'user.saved.id', - migration: true, - }, - { - name: 'ogid', - type: 'alias', - path: 'user.owner.group.id', - migration: true, - }, - { - name: 'ouid', - type: 'alias', - path: 'user.owner.id', - migration: true, - }, - { - name: 'comm', - type: 'alias', - path: 'process.name', - migration: true, - }, - { - name: 'exe', - type: 'alias', - path: 'process.executable', - migration: true, - }, - { - name: 'terminal', - type: 'alias', - path: 'user.terminal', - migration: true, - }, - { - name: 'msg', - type: 'alias', - path: 'message', - migration: true, - }, - { - name: 'src', - type: 'alias', - path: 'source.address', - migration: true, - }, - { - name: 'dst', - type: 'alias', - path: 'destination.address', - migration: true, + name: 'text', + type: 'text', + norms: false, + default_field: false, }, ], + description: 'Absolute path to the process executable.', + example: '/usr/bin/ssh', }, - ], - }, - ], - }, - { - key: 'elasticsearch', - title: 'elasticsearch', - description: 'elasticsearch Module', - fields: [ - { - name: 'elasticsearch', - type: 'group', - description: '', - fields: [ { - name: 'component', - description: 'Elasticsearch component from where the log event originated', - example: 'o.e.c.m.MetaDataCreateIndexService', - type: 'keyword', + name: 'exit_code', + level: 'extended', + type: 'long', + description: + 'The exit code of the process, if this is a termination event.\n\nThe field should be absent if there is no exit code for the event (e.g. process\nstart).', + example: 137, + default_field: false, }, { - name: 'cluster.uuid', - description: 'UUID of the cluster', - example: 'GmvrbHlNTiSVYiPf8kxg9g', + name: 'hash.md5', + level: 'extended', type: 'keyword', + ignore_above: 1024, + description: 'MD5 hash.', }, { - name: 'cluster.name', - description: 'Name of the cluster', - example: 'docker-cluster', + name: 'hash.sha1', + level: 'extended', type: 'keyword', + ignore_above: 1024, + description: 'SHA1 hash.', }, { - name: 'node.id', - description: 'ID of the node', - example: 'DSiWcTyeThWtUXLB9J0BMw', + name: 'hash.sha256', + level: 'extended', type: 'keyword', + ignore_above: 1024, + description: 'SHA256 hash.', }, { - name: 'node.name', - description: 'Name of the node', - example: 'vWNJsZ3', + name: 'hash.sha512', + level: 'extended', type: 'keyword', + ignore_above: 1024, + description: 'SHA512 hash.', }, { - name: 'index.name', - description: 'Index name', - example: 'filebeat-test-input', + name: 'name', + level: 'extended', type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Process name.\n\nSometimes called program name or similar.', + example: 'ssh', }, { - name: 'index.id', - description: 'Index id', - example: 'aOGgDwbURfCV57AScqbCgw', + name: 'parent.args', + level: 'extended', type: 'keyword', + ignore_above: 1024, + description: + 'Array of process arguments.\n\nMay be filtered to protect sensitive information.', + example: ['ssh', '-l', 'user', '10.0.0.16'], + default_field: false, }, { - name: 'shard.id', - description: 'Id of the shard', - example: '0', + name: 'parent.args_count', + level: 'extended', + type: 'long', + description: + 'Length of the process.args array.\n\nThis field can be useful for querying or performing bucket analysis on how\nmany arguments were provided to start a process. More arguments may be an\nindication of suspicious activity.', + example: 4, + default_field: false, + }, + { + name: 'parent.code_signature.exists', + level: 'core', + type: 'boolean', + description: 'Boolean to capture if a signature is present.', + example: 'true', + default_field: false, + }, + { + name: 'parent.code_signature.status', + level: 'extended', type: 'keyword', + ignore_above: 1024, + description: + 'Additional information about the certificate status.\n\nThis is useful for logging cryptographic errors with the certificate validity\nor trust status. Leave unpopulated if the validity or trust of the certificate\nwas unchecked.', + example: 'ERROR_UNTRUSTED_ROOT', + default_field: false, }, { - name: 'audit', - type: 'group', - description: '', - fields: [ - { - name: 'layer', - description: - 'The layer from which this event originated: rest, transport or ip_filter', - example: 'rest', - type: 'keyword', - }, - { - name: 'origin.type', - description: - 'Where the request originated: rest (request originated from a REST API request), transport (request was received on the transport channel), local_node (the local node issued the request)', - example: 'local_node', - type: 'keyword', - }, - { - name: 'realm', - description: 'The authentication realm the authentication was validated against', - example: 'default_file', - type: 'keyword', - }, - { - name: 'user.realm', - description: "The user's authentication realm, if authenticated", - example: 'active_directory', - type: 'keyword', - }, - { - name: 'user.roles', - description: 'Roles to which the principal belongs', - example: ['kibana_admin', 'beats_admin'], - type: 'keyword', - }, - { - name: 'action', - description: 'The name of the action that was executed', - example: 'cluster:monitor/main', - type: 'keyword', - }, - { - name: 'url.params', - description: 'REST URI parameters', - example: '{username=jacknich2}', - }, - { - name: 'indices', - description: 'Indices accessed by action', - example: ['foo-2019.01.04', 'foo-2019.01.03', 'foo-2019.01.06'], - type: 'keyword', - }, - { - name: 'request.id', - description: 'Unique ID of request', - example: 'WzL_kb6VSvOhAq0twPvHOQ', - type: 'keyword', - }, - { - name: 'request.name', - description: 'The type of request that was executed', - example: 'ClearScrollRequest', - type: 'keyword', - }, - { - name: 'request_body', - type: 'alias', - path: 'http.request.body.content', - migration: true, - }, - { - name: 'event_type', - type: 'alias', - path: 'event.type', - migration: true, - }, + name: 'parent.code_signature.subject_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Subject name of the code signer', + example: 'Microsoft Corporation', + default_field: false, + }, + { + name: 'parent.code_signature.trusted', + level: 'extended', + type: 'boolean', + description: + 'Stores the trust status of the certificate chain.\n\nValidating the trust of the certificate chain may be complicated, and this\nfield should only be populated by tools that actively check the status.', + example: 'true', + default_field: false, + }, + { + name: 'parent.code_signature.valid', + level: 'extended', + type: 'boolean', + description: + 'Boolean to capture if the digital signature is verified against\nthe binary content.\n\nLeave unpopulated if a certificate was unchecked.', + example: 'true', + default_field: false, + }, + { + name: 'parent.command_line', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ { - name: 'origin_address', - type: 'alias', - path: 'source.ip', - migration: true, + name: 'text', + type: 'text', + norms: false, }, + ], + description: + 'Full command line that started the process, including the absolute\npath to the executable, and all arguments.\n\nSome arguments may be filtered to protect sensitive information.', + example: '/usr/bin/ssh -l user 10.0.0.16', + default_field: false, + }, + { + name: 'parent.entity_id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Unique identifier for the process.\n\nThe implementation of this is specified by the data source, but some examples\nof what could be used here are a process-generated UUID, Sysmon Process GUIDs,\nor a hash of some uniquely identifying components of a process.\n\nConstructing a globally unique identifier is a common practice to mitigate\nPID reuse as well as to identify a specific process over time, across multiple\nmonitored hosts.', + example: 'c2c455d9f99375d', + default_field: false, + }, + { + name: 'parent.executable', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ { - name: 'uri', - type: 'alias', - path: 'url.original', - migration: true, + name: 'text', + type: 'text', + norms: false, }, + ], + description: 'Absolute path to the process executable.', + example: '/usr/bin/ssh', + default_field: false, + }, + { + name: 'parent.exit_code', + level: 'extended', + type: 'long', + description: + 'The exit code of the process, if this is a termination event.\n\nThe field should be absent if there is no exit code for the event (e.g. process\nstart).', + example: 137, + default_field: false, + }, + { + name: 'parent.hash.md5', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'MD5 hash.', + default_field: false, + }, + { + name: 'parent.hash.sha1', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'SHA1 hash.', + default_field: false, + }, + { + name: 'parent.hash.sha256', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'SHA256 hash.', + default_field: false, + }, + { + name: 'parent.hash.sha512', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'SHA512 hash.', + default_field: false, + }, + { + name: 'parent.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ { - name: 'principal', - type: 'alias', - path: 'user.name', - migration: true, + name: 'text', + type: 'text', + norms: false, }, ], + description: 'Process name.\n\nSometimes called program name or similar.', + example: 'ssh', + default_field: false, }, { - name: 'deprecation', - type: 'group', - description: '', + name: 'parent.pgid', + level: 'extended', + type: 'long', + format: 'string', + description: 'Identifier of the group of processes the process belongs to.', + default_field: false, }, { - name: 'gc', - type: 'group', - description: 'GC fileset fields.', - fields: [ + name: 'parent.pid', + level: 'core', + type: 'long', + format: 'string', + description: 'Process id.', + example: 4242, + default_field: false, + }, + { + name: 'parent.ppid', + level: 'extended', + type: 'long', + format: 'string', + description: "Parent process' pid.", + example: 4241, + default_field: false, + }, + { + name: 'parent.start', + level: 'extended', + type: 'date', + description: 'The time the process started.', + example: '2016-05-23T08:05:34.853Z', + default_field: false, + }, + { + name: 'parent.thread.id', + level: 'extended', + type: 'long', + format: 'string', + description: 'Thread ID.', + example: 4242, + default_field: false, + }, + { + name: 'parent.thread.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Thread name.', + example: 'thread-0', + default_field: false, + }, + { + name: 'parent.title', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ { - name: 'phase', - type: 'group', - description: 'Fields specific to GC phase.', - fields: [ - { - name: 'name', - type: 'keyword', - description: 'Name of the GC collection phase.', - }, - { - name: 'duration_sec', - type: 'float', - description: 'Collection phase duration according to the Java virtual machine.', - }, - { - name: 'scrub_symbol_table_time_sec', - type: 'float', - description: 'Pause time in seconds cleaning up symbol tables.', - }, - { - name: 'scrub_string_table_time_sec', - type: 'float', - description: 'Pause time in seconds cleaning up string tables.', - }, - { - name: 'weak_refs_processing_time_sec', - type: 'float', - description: 'Time spent processing weak references in seconds.', - }, - { - name: 'parallel_rescan_time_sec', - type: 'float', - description: - 'Time spent in seconds marking live objects while application is stopped.', - }, - { - name: 'class_unload_time_sec', - type: 'float', - description: 'Time spent unloading unused classes in seconds.', - }, - { - name: 'cpu_time', - type: 'group', - description: 'Process CPU time spent performing collections.', - fields: [ - { - name: 'user_sec', - type: 'float', - description: 'CPU time spent outside the kernel.', - }, - { - name: 'sys_sec', - type: 'float', - description: 'CPU time spent inside the kernel.', - }, - { - name: 'real_sec', - type: 'float', - description: - 'Total elapsed CPU time spent to complete the collection from start to finish.', - }, - ], - }, - ], + name: 'text', + type: 'text', + norms: false, }, + ], + description: + 'Process title.\n\nThe proctitle, some times the same as process name. Can also be different:\nfor example a browser setting its title to the web page currently opened.', + default_field: false, + }, + { + name: 'parent.uptime', + level: 'extended', + type: 'long', + description: 'Seconds the process has been up.', + example: 1325, + default_field: false, + }, + { + name: 'parent.working_directory', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ { - name: 'jvm_runtime_sec', - type: 'float', - description: 'The time from JVM start up in seconds, as a floating point number.', - }, - { - name: 'threads_total_stop_time_sec', - type: 'float', - description: 'Garbage collection threads total stop time seconds.', - }, - { - name: 'stopping_threads_time_sec', - type: 'float', - description: 'Time took to stop threads seconds.', - }, - { - name: 'tags', - type: 'keyword', - description: 'GC logging tags.', - }, - { - name: 'heap', - type: 'group', - description: 'Heap allocation and total size.', - fields: [ - { - name: 'size_kb', - type: 'integer', - description: 'Total heap size in kilobytes.', - }, - { - name: 'used_kb', - type: 'integer', - description: 'Used heap in kilobytes.', - }, - ], - }, - { - name: 'old_gen', - type: 'group', - description: 'Old generation occupancy and total size.', - fields: [ - { - name: 'size_kb', - type: 'integer', - description: 'Total size of old generation in kilobytes.', - }, - { - name: 'used_kb', - type: 'integer', - description: 'Old generation occupancy in kilobytes.', - }, - ], - }, - { - name: 'young_gen', - type: 'group', - description: 'Young generation occupancy and total size.', - fields: [ - { - name: 'size_kb', - type: 'integer', - description: 'Total size of young generation in kilobytes.', - }, - { - name: 'used_kb', - type: 'integer', - description: 'Young generation occupancy in kilobytes.', - }, - ], + name: 'text', + type: 'text', + norms: false, }, ], + description: 'The working directory of the process.', + example: '/home/alice', + default_field: false, }, { - name: 'server', - description: 'Server log file', - type: 'group', - fields: [ - { - name: 'stacktrace', - description: 'Stack trace in case of errors', - index: false, - }, - { - name: 'gc', - description: 'GC log', - type: 'group', - fields: [ - { - name: 'young', - description: 'Young GC', - example: '', - type: 'group', - fields: [ - { - name: 'one', - description: '', - example: '', - type: 'long', - }, - { - name: 'two', - description: '', - example: '', - type: 'long', - }, - ], - }, - { - name: 'overhead_seq', - description: 'Sequence number', - example: 3449992, - type: 'long', - }, - { - name: 'collection_duration.ms', - description: 'Time spent in GC, in milliseconds', - example: 1600, - type: 'float', - }, - { - name: 'observation_duration.ms', - description: 'Total time over which collection was observed, in milliseconds', - example: 1800, - type: 'float', - }, - ], - }, - ], + name: 'pe.company', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal company name of the file, provided at compile-time.', + example: 'Microsoft Corporation', + default_field: false, }, { - name: 'slowlog', - description: 'Slowlog events from Elasticsearch', - example: - '[2018-06-29T10:06:14,933][INFO ][index.search.slowlog.query] [v_VJhjV] [metricbeat-6.3.0-2018.06.26][0] took[4.5ms], took_millis[4], total_hits[19435], types[], stats[], search_type[QUERY_THEN_FETCH], total_shards[1], source[{"query":{"match_all":{"boost":1.0}}}],', - type: 'group', - fields: [ - { - name: 'logger', - description: 'Logger name', - example: 'index.search.slowlog.fetch', - type: 'keyword', - }, - { - name: 'took', - description: 'Time it took to execute the query', - example: '300ms', - type: 'keyword', - }, - { - name: 'types', - description: 'Types', - example: '', - type: 'keyword', - }, - { - name: 'stats', - description: 'Stats groups', - example: 'group1', - type: 'keyword', - }, - { - name: 'search_type', - description: 'Search type', - example: 'QUERY_THEN_FETCH', - type: 'keyword', - }, - { - name: 'source_query', - description: 'Slow query', - example: '{"query":{"match_all":{"boost":1.0}}}', - type: 'keyword', - }, - { - name: 'extra_source', - description: 'Extra source information', - example: '', - type: 'keyword', - }, - { - name: 'total_hits', - description: 'Total hits', - example: 42, - type: 'keyword', - }, - { - name: 'total_shards', - description: 'Total queried shards', - example: 22, - type: 'keyword', - }, - { - name: 'routing', - description: 'Routing', - example: 's01HZ2QBk9jw4gtgaFtn', - type: 'keyword', - }, - { - name: 'id', - description: 'Id', - example: '', - type: 'keyword', - }, - { - name: 'type', - description: 'Type', - example: 'doc', - type: 'keyword', - }, - ], + name: 'pe.description', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal description of the file, provided at compile-time.', + example: 'Paint', + default_field: false, }, - ], - }, - ], - }, - { - key: 'haproxy', - title: 'haproxy', - description: 'haproxy Module', - fields: [ - { - name: 'haproxy', - type: 'group', - description: '', - fields: [ { - name: 'frontend_name', - description: - 'Name of the frontend (or listener) which received and processed the connection.', + name: 'pe.file_version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal version of the file, provided at compile-time.', + example: '6.3.9600.17415', + default_field: false, }, { - name: 'backend_name', - description: - 'Name of the backend (or listener) which was selected to manage the connection to the server.', + name: 'pe.original_file_name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal name of the file, provided at compile-time.', + example: 'MSPAINT.EXE', + default_field: false, }, { - name: 'server_name', - description: 'Name of the last server to which the connection was sent.', + name: 'pe.product', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal product name of the file, provided at compile-time.', + example: 'Microsoft® Windows® Operating System', + default_field: false, }, { - name: 'total_waiting_time_ms', - description: 'Total time in milliseconds spent waiting in the various queues', + name: 'pgid', + level: 'extended', type: 'long', + format: 'string', + description: 'Identifier of the group of processes the process belongs to.', }, { - name: 'connection_wait_time_ms', - description: - 'Total time in milliseconds spent waiting for the connection to establish to the final server', + name: 'pid', + level: 'core', type: 'long', + format: 'string', + description: 'Process id.', + example: 4242, }, { - name: 'bytes_read', - description: 'Total number of bytes transmitted to the client when the log is emitted.', + name: 'ppid', + level: 'extended', type: 'long', + format: 'string', + description: "Parent process' pid.", + example: 4241, }, { - name: 'time_queue', - description: 'Total time in milliseconds spent waiting in the various queues.', - type: 'long', + name: 'start', + level: 'extended', + type: 'date', + description: 'The time the process started.', + example: '2016-05-23T08:05:34.853Z', }, { - name: 'time_backend_connect', - description: - 'Total time in milliseconds spent waiting for the connection to establish to the final server, including retries.', + name: 'thread.id', + level: 'extended', type: 'long', + format: 'string', + description: 'Thread ID.', + example: 4242, }, { - name: 'server_queue', - description: - 'Total number of requests which were processed before this one in the server queue.', - type: 'long', + name: 'thread.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Thread name.', + example: 'thread-0', }, { - name: 'backend_queue', - description: - "Total number of requests which were processed before this one in the backend's global queue.", - type: 'long', - }, - { - name: 'bind_name', - description: 'Name of the listening address which received the connection.', - }, - { - name: 'error_message', - description: 'Error message logged by HAProxy in case of error.', - type: 'text', - }, - { - name: 'source', + name: 'title', + level: 'extended', type: 'keyword', - description: 'The HAProxy source of the log', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: + 'Process title.\n\nThe proctitle, some times the same as process name. Can also be different:\nfor example a browser setting its title to the web page currently opened.', }, { - name: 'termination_state', - description: 'Condition the session was in when the session ended.', + name: 'uptime', + level: 'extended', + type: 'long', + description: 'Seconds the process has been up.', + example: 1325, }, { - name: 'mode', + name: 'working_directory', + level: 'extended', type: 'keyword', - description: 'mode that the frontend is operating (TCP or HTTP)', - }, - { - name: 'connections', - description: 'Contains various counts of connections active in the process.', - type: 'group', - fields: [ - { - name: 'active', - description: - 'Total number of concurrent connections on the process when the session was logged.', - type: 'long', - }, - { - name: 'frontend', - description: - 'Total number of concurrent connections on the frontend when the session was logged.', - type: 'long', - }, - { - name: 'backend', - description: - 'Total number of concurrent connections handled by the backend when the session was logged.', - type: 'long', - }, - { - name: 'server', - description: - 'Total number of concurrent connections still active on the server when the session was logged.', - type: 'long', - }, + ignore_above: 1024, + multi_fields: [ { - name: 'retries', - description: - 'Number of connection retries experienced by this session when trying to connect to the server.', - type: 'long', + name: 'text', + type: 'text', + norms: false, + default_field: false, }, ], + description: 'The working directory of the process.', + example: '/home/alice', }, + ], + }, + { + name: 'registry', + title: 'Registry', + group: 2, + description: 'Fields related to Windows Registry operations.', + type: 'group', + fields: [ { - name: 'client', - description: 'Information about the client doing the request', - type: 'group', - fields: [ - { - name: 'ip', - type: 'alias', - path: 'source.address', - migration: true, - }, - { - name: 'port', - type: 'alias', - path: 'source.port', - migration: true, - }, - ], + name: 'data.bytes', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Original bytes written with base64 encoding.\n\nFor Windows registry operations, such as SetValueEx and RegQueryValueEx, this\ncorresponds to the data pointed by `lp_data`. This is optional but provides\nbetter recoverability and should be populated for REG_BINARY encoded values.', + example: 'ZQBuAC0AVQBTAAAAZQBuAAAAAAA=', + default_field: false, }, { - name: 'process_name', - type: 'alias', - path: 'process.name', - migration: true, + name: 'data.strings', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'Content when writing string types.\n\nPopulated as an array when writing string data to the registry. For single\nstring registry types (REG_SZ, REG_EXPAND_SZ), this should be an array with\none string. For sequences of string with REG_MULTI_SZ, this array will be\nvariable length. For numeric data, such as REG_DWORD and REG_QWORD, this should\nbe populated with the decimal representation (e.g `"1"`).', + example: '["C:\\rta\\red_ttp\\bin\\myapp.exe"]', + default_field: false, }, { - name: 'pid', - type: 'alias', - path: 'process.pid', - migration: true, + name: 'data.type', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Standard registry type for encoding contents', + example: 'REG_SZ', + default_field: false, }, { - name: 'destination', - description: 'Destination information', - type: 'group', - fields: [ - { - name: 'port', - type: 'alias', - path: 'destination.port', - migration: true, - }, - { - name: 'ip', - type: 'alias', - path: 'destination.ip', - migration: true, - }, - ], + name: 'hive', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Abbreviated name for the hive.', + example: 'HKLM', + default_field: false, }, { - name: 'geoip', - type: 'group', - description: - 'Contains GeoIP information gathered based on the client.ip field. Only present if the GeoIP Elasticsearch plugin is available and used.', - fields: [ - { - name: 'continent_name', - type: 'alias', - path: 'source.geo.continent_name', - migration: true, - }, - { - name: 'country_iso_code', - type: 'alias', - path: 'source.geo.country_iso_code', - migration: true, - }, - { - name: 'location', - type: 'alias', - path: 'source.geo.location', - migration: true, - }, - { - name: 'region_name', - type: 'alias', - path: 'source.geo.region_name', - migration: true, - }, - { - name: 'city_name', - type: 'alias', - path: 'source.geo.city_name', - migration: true, - }, - { - name: 'region_iso_code', - type: 'alias', - path: 'source.geo.region_iso_code', - migration: true, - }, - ], + name: 'key', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Hive-relative path of keys.', + example: + 'SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Image File Execution Options\\winword.exe', + default_field: false, }, { - name: 'http', - description: 'Please add description', - type: 'group', - fields: [ - { - name: 'response', - description: 'Fields related to the HTTP response', - type: 'group', - fields: [ - { - name: 'captured_cookie', - description: - 'Optional "name=value" entry indicating that the client had this cookie in the response.', - }, - { - name: 'captured_headers', - description: - 'List of headers captured in the response due to the presence of the "capture response header" statement in the frontend.', - type: 'keyword', - }, - { - name: 'status_code', - type: 'alias', - path: 'http.response.status_code', - migration: true, - }, - ], - }, - { - name: 'request', - description: 'Fields related to the HTTP request', - type: 'group', - fields: [ - { - name: 'captured_cookie', - description: - 'Optional "name=value" entry indicating that the server has returned a cookie with its request.', - }, - { - name: 'captured_headers', - description: - 'List of headers captured in the request due to the presence of the "capture request header" statement in the frontend.', - type: 'keyword', - }, - { - name: 'raw_request_line', - description: - 'Complete HTTP request line, including the method, request and HTTP version string.', - type: 'keyword', - }, - { - name: 'time_wait_without_data_ms', - description: - 'Total time in milliseconds spent waiting for the server to send a full HTTP response, not counting data.', - type: 'long', - }, - { - name: 'time_wait_ms', - description: - 'Total time in milliseconds spent waiting for a full HTTP request from the client (not counting body) after the first byte was received.', - type: 'long', - }, - ], - }, - ], + name: 'path', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Full path, including hive, key and value', + example: + 'HKLM\\SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Image File Execution\nOptions\\winword.exe\\Debugger', + default_field: false, }, { - name: 'tcp', - description: 'TCP log format', - type: 'group', - fields: [ - { - name: 'connection_waiting_time_ms', - type: 'long', - description: - 'Total time in milliseconds elapsed between the accept and the last close', - }, - ], + name: 'value', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the value written.', + example: 'Debugger', + default_field: false, }, ], }, - ], - }, - { - key: 'icinga', - title: 'Icinga', - description: 'Icinga Module', - fields: [ { - name: 'icinga', + name: 'related', + title: 'Related', + group: 2, + description: + 'This field set is meant to facilitate pivoting around a piece of\ndata.\n\nSome pieces of information can be seen in many places in an ECS event. To facilitate\nsearching for them, store an array of all seen values to their corresponding\nfield in `related.`.\n\nA concrete example is IP addresses, which can be under host, observer, source,\ndestination, client, server, and network.forwarded_ip. If you append all IPs\nto `related.ip`, you can then search for a given IP trivially, no matter where\nit appeared, by querying `related.ip:192.0.2.15`.', type: 'group', - description: '', fields: [ { - name: 'debug', - type: 'group', - description: 'Contains fields for the Icinga debug logs.', - fields: [ - { - name: 'facility', - type: 'keyword', - description: 'Specifies what component of Icinga logged the message.', - }, - { - name: 'severity', - type: 'alias', - path: 'log.level', - migration: true, - }, - { - name: 'message', - type: 'alias', - path: 'message', - migration: true, - }, - ], + name: 'hash', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + "All the hashes seen on your event. Populating this field, then\nusing it to search for hashes can help in situations where you're unsure what\nthe hash algorithm is (and therefore which key name to search).", + default_field: false, }, { - name: 'main', - type: 'group', - description: 'Contains fields for the Icinga main logs.', - fields: [ - { - name: 'facility', - type: 'keyword', - description: 'Specifies what component of Icinga logged the message.', - }, - { - name: 'severity', - type: 'alias', - path: 'log.level', - migration: true, - }, - { - name: 'message', - type: 'alias', - path: 'message', - migration: true, - }, - ], + name: 'ip', + level: 'extended', + type: 'ip', + description: 'All of the IPs seen on your event.', }, { - name: 'startup', - type: 'group', - description: 'Contains fields for the Icinga startup logs.', - fields: [ - { - name: 'facility', - type: 'keyword', - description: 'Specifies what component of Icinga logged the message.', - }, - { - name: 'severity', - type: 'alias', - path: 'log.level', - migration: true, - }, - { - name: 'message', - type: 'alias', - path: 'message', - migration: true, - }, - ], + name: 'user', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'All the user names seen on your event.', + default_field: false, }, ], }, - ], - }, - { - key: 'iis', - title: 'IIS', - description: 'Module for parsing IIS log files.', - fields: [ { - name: 'iis', + name: 'rule', + title: 'Rule', + group: 2, + description: + 'Rule fields are used to capture the specifics of any observer or\nagent rules that generate alerts or other notable events.\n\nExamples of data sources that would populate the rule fields include: network\nadmission control platforms, network or host IDS/IPS, network firewalls, web\napplication firewalls, url filters, endpoint detection and response (EDR) systems,\netc.', type: 'group', - description: 'Fields from IIS log files.', fields: [ { - name: 'access', - type: 'group', - description: 'Contains fields for IIS access logs.', - fields: [ - { - name: 'sub_status', - type: 'long', - description: 'The HTTP substatus code.', - }, - { - name: 'win32_status', - type: 'long', - description: 'The Windows status code.', - }, - { - name: 'site_name', - type: 'keyword', - description: 'The site name and instance number.', - }, - { - name: 'server_name', - type: 'keyword', - description: 'The name of the server on which the log file entry was generated.', - }, - { - name: 'cookie', - type: 'keyword', - description: 'The content of the cookie sent or received, if any.', - }, - { - name: 'body_received.bytes', - type: 'alias', - path: 'http.request.body.bytes', - migration: true, - }, - { - name: 'body_sent.bytes', - type: 'alias', - path: 'http.response.body.bytes', - migration: true, - }, - { - name: 'server_ip', - type: 'alias', - path: 'destination.address', - migration: true, - }, - { - name: 'method', - type: 'alias', - path: 'http.request.method', - migration: true, - }, - { - name: 'url', - type: 'alias', - path: 'url.path', - migration: true, - }, - { - name: 'query_string', - type: 'alias', - path: 'url.query', - migration: true, - }, - { - name: 'port', - type: 'alias', - path: 'destination.port', - migration: true, - }, - { - name: 'user_name', - type: 'alias', - path: 'user.name', - migration: true, - }, - { - name: 'remote_ip', - type: 'alias', - path: 'source.address', - migration: true, - }, - { - name: 'referrer', - type: 'alias', - path: 'http.request.referrer', - migration: true, - }, - { - name: 'response_code', - type: 'alias', - path: 'http.response.status_code', - migration: true, - }, - { - name: 'http_version', - type: 'alias', - path: 'http.version', - migration: true, - }, + name: 'author', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name, organization, or pseudonym of the author or authors who created\nthe rule used to generate this event.', + example: ['Star-Lord'], + default_field: false, + }, + { + name: 'category', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'A categorization value keyword used by the entity using the rule\nfor detection of this event.', + example: 'Attempted Information Leak', + default_field: false, + }, + { + name: 'description', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'The description of the rule generating the event.', + example: 'Block requests to public DNS over HTTPS / TLS protocols', + default_field: false, + }, + { + name: 'id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'A rule ID that is unique within the scope of an agent, observer,\nor other entity using the rule for detection of this event.', + example: 101, + default_field: false, + }, + { + name: 'license', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the license under which the rule used to generate this\nevent is made available.', + example: 'Apache 2.0', + default_field: false, + }, + { + name: 'name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'The name of the rule or signature generating the event.', + example: 'BLOCK_DNS_over_TLS', + default_field: false, + }, + { + name: 'reference', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Reference URL to additional information about the rule used to\ngenerate this event.\n\nThe URL can point to the vendor documentation about the rule. If that is\nnot available, it can also be a link to a more general page describing this\ntype of alert.', + example: 'https://en.wikipedia.org/wiki/DNS_over_TLS', + default_field: false, + }, + { + name: 'ruleset', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the ruleset, policy, group, or parent category in which\nthe rule used to generate this event is a member.', + example: 'Standard_Protocol_Filters', + default_field: false, + }, + { + name: 'uuid', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'A rule ID that is unique within the scope of a set or group of\nagents, observers, or other entities using the rule for detection of this\nevent.', + example: 1100110011, + default_field: false, + }, + { + name: 'version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'The version / revision of the rule being used for analysis.', + example: 1.1, + default_field: false, + }, + ], + }, + { + name: 'server', + title: 'Server', + group: 2, + description: + 'A Server is defined as the responder in a network connection for\nevents regarding sessions, connections, or bidirectional flow records.\n\nFor TCP events, the server is the receiver of the initial SYN packet(s) of the\nTCP connection. For other protocols, the server is generally the responder in\nthe network transaction. Some systems actually use the term "responder" to refer\nthe server in TCP connections. The server fields describe details about the\nsystem acting as the server in the network event. Server fields are usually\npopulated in conjunction with client fields. Server fields are generally not\npopulated for packet-level events.\n\nClient / server representations can add semantic context to an exchange, which\nis helpful to visualize the data in certain situations. If your context falls\nin that category, you should still ensure that source and destination are filled\nappropriately.', + type: 'group', + fields: [ + { + name: 'address', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Some event server addresses are defined ambiguously. The event\nwill sometimes list an IP, a domain or a unix socket. You should always store\nthe raw address in the `.address` field.\n\nThen it should be duplicated to `.ip` or `.domain`, depending on which one\nit is.', + }, + { + name: 'as.number', + level: 'extended', + type: 'long', + description: + 'Unique number allocated to the autonomous system. The autonomous\nsystem number (ASN) uniquely identifies each network on the Internet.', + example: 15169, + }, + { + name: 'as.organization.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ { - name: 'hostname', - type: 'alias', - path: 'host.hostname', - migration: true, - }, - { - name: 'user_agent', - type: 'group', - fields: [ - { - name: 'device', - type: 'alias', - path: 'user_agent.device.name', - migration: true, - }, - { - name: 'name', - type: 'alias', - path: 'user_agent.name', - migration: true, - }, - { - name: 'os', - type: 'alias', - path: 'user_agent.os.full_name', - migration: true, - }, - { - name: 'os_name', - type: 'alias', - path: 'user_agent.os.name', - migration: true, - }, - { - name: 'original', - type: 'alias', - path: 'user_agent.original', - migration: true, - }, - ], - }, - { - name: 'geoip', - type: 'group', - fields: [ - { - name: 'continent_name', - type: 'alias', - path: 'source.geo.continent_name', - migration: true, - }, - { - name: 'country_iso_code', - type: 'alias', - path: 'source.geo.country_iso_code', - migration: true, - }, - { - name: 'location', - type: 'alias', - path: 'source.geo.location', - migration: true, - }, - { - name: 'region_name', - type: 'alias', - path: 'source.geo.region_name', - migration: true, - }, - { - name: 'city_name', - type: 'alias', - path: 'source.geo.city_name', - migration: true, - }, - { - name: 'region_iso_code', - type: 'alias', - path: 'source.geo.region_iso_code', - migration: true, - }, - ], + name: 'text', + type: 'text', + norms: false, + default_field: false, }, ], + description: 'Organization name.', + example: 'Google LLC', }, { - name: 'error', - type: 'group', - description: 'Contains fields for IIS error logs.', - fields: [ - { - name: 'reason_phrase', - type: 'keyword', - description: 'The HTTP reason phrase.', - }, - { - name: 'queue_name', - type: 'keyword', - description: 'The IIS application pool name.', - }, - { - name: 'remote_ip', - type: 'alias', - path: 'source.address', - migration: true, - }, - { - name: 'remote_port', - type: 'alias', - path: 'source.port', - migration: true, - }, - { - name: 'server_ip', - type: 'alias', - path: 'destination.address', - migration: true, - }, - { - name: 'server_port', - type: 'alias', - path: 'destination.port', - migration: true, - }, - { - name: 'http_version', - type: 'alias', - path: 'http.version', - migration: true, - }, - { - name: 'method', - type: 'alias', - path: 'http.request.method', - migration: true, - }, - { - name: 'url', - type: 'alias', - path: 'url.original', - migration: true, - }, - { - name: 'response_code', - type: 'alias', - path: 'http.response.status_code', - migration: true, - }, - { - name: 'geoip', - type: 'group', - fields: [ - { - name: 'continent_name', - type: 'alias', - path: 'source.geo.continent_name', - migration: true, - }, - { - name: 'country_iso_code', - type: 'alias', - path: 'source.geo.country_iso_code', - migration: true, - }, - { - name: 'location', - type: 'alias', - path: 'source.geo.location', - migration: true, - }, - { - name: 'region_name', - type: 'alias', - path: 'source.geo.region_name', - migration: true, - }, - { - name: 'city_name', - type: 'alias', - path: 'source.geo.city_name', - migration: true, - }, - { - name: 'region_iso_code', - type: 'alias', - path: 'source.geo.region_iso_code', - migration: true, - }, - ], - }, - ], + name: 'bytes', + level: 'core', + type: 'long', + format: 'bytes', + description: 'Bytes sent from the server to the client.', + example: 184, }, - ], - }, - ], - }, - { - key: 'kafka', - title: 'Kafka', - description: 'Kafka module', - fields: [ - { - name: 'kafka', - type: 'group', - description: '', - fields: [ { - name: 'log', - type: 'group', - description: 'Kafka log lines.', - fields: [ - { - name: 'level', - type: 'alias', - path: 'log.level', - migration: true, - }, - { - name: 'message', - type: 'alias', - path: 'message', - migration: true, - }, - { - name: 'component', - type: 'keyword', - description: 'Component the log is coming from.', - }, - { - name: 'class', - type: 'keyword', - description: 'Java class the log is coming from.', - }, - { - name: 'trace', - type: 'group', - description: 'Trace in the log line.', - fields: [ - { - name: 'class', - type: 'keyword', - description: 'Java class the trace is coming from.', - }, - { - name: 'message', - type: 'text', - description: 'Message part of the trace.', - }, - ], - }, - ], + name: 'domain', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Server domain.', }, - ], - }, - ], - }, - { - key: 'kibana', - title: 'kibana', - description: 'kibana Module', - fields: [ - { - name: 'kibana', - type: 'group', - description: '', - fields: [ { - name: 'log', - type: 'group', - description: 'Kafka log lines.', - fields: [ - { - name: 'tags', - type: 'keyword', - description: 'Kibana logging tags.', - }, - { - name: 'state', - type: 'keyword', - description: 'Current state of Kibana.', - }, - { - name: 'meta', - type: 'object', - object_type: 'keyword', - }, - { - name: 'kibana.log.meta.req.headers.referer', - type: 'alias', - path: 'http.request.referrer', - migration: true, - }, - { - name: 'kibana.log.meta.req.referer', - type: 'alias', - path: 'http.request.referrer', - migration: true, - }, - { - name: 'kibana.log.meta.req.headers.user-agent', - type: 'alias', - path: 'user_agent.original', - migration: true, - }, - { - name: 'kibana.log.meta.req.remoteAddress', - type: 'alias', - path: 'source.address', - migration: true, - }, - { - name: 'kibana.log.meta.req.url', - type: 'alias', - path: 'url.original', - migration: true, - }, + name: 'geo.city_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'City name.', + example: 'Montreal', + }, + { + name: 'geo.continent_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the continent.', + example: 'North America', + }, + { + name: 'geo.country_iso_code', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Country ISO code.', + example: 'CA', + }, + { + name: 'geo.country_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Country name.', + example: 'Canada', + }, + { + name: 'geo.location', + level: 'core', + type: 'geo_point', + description: 'Longitude and latitude.', + example: '{ "lon": -73.614830, "lat": 45.505918 }', + }, + { + name: 'geo.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'User-defined description of a location, at the level of granularity\nthey care about.\n\nCould be the name of their data centers, the floor number, if this describes\na local physical entity, city names.\n\nNot typically used in automated geolocation.', + example: 'boston-dc', + }, + { + name: 'geo.region_iso_code', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Region ISO code.', + example: 'CA-QC', + }, + { + name: 'geo.region_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Region name.', + example: 'Quebec', + }, + { + name: 'ip', + level: 'core', + type: 'ip', + description: + 'IP address of the server.\n\nCan be one or multiple IPv4 or IPv6 addresses.', + }, + { + name: 'mac', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'MAC address of the server.', + }, + { + name: 'nat.ip', + level: 'extended', + type: 'ip', + description: + 'Translated ip of destination based NAT sessions (e.g. internet\nto private DMZ)\n\nTypically used with load balancers, firewalls, or routers.', + }, + { + name: 'nat.port', + level: 'extended', + type: 'long', + format: 'string', + description: + 'Translated port of destination based NAT sessions (e.g. internet\nto private DMZ)\n\nTypically used with load balancers, firewalls, or routers.', + }, + { + name: 'packets', + level: 'core', + type: 'long', + description: 'Packets sent from the server to the client.', + example: 12, + }, + { + name: 'port', + level: 'core', + type: 'long', + format: 'string', + description: 'Port of the server.', + }, + { + name: 'registered_domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The highest registered server domain, stripped of the subdomain.\n\nFor example, the registered domain for "foo.google.com" is "google.com".\n\nThis value can be determined precisely with a list like the public suffix\nlist (http://publicsuffix.org). Trying to approximate this by simply taking\nthe last two labels will not work well for TLDs such as "co.uk".', + example: 'google.com', + }, + { + name: 'top_level_domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The effective top level domain (eTLD), also known as the domain\nsuffix, is the last part of the domain name. For example, the top level domain\nfor google.com is "com".\n\nThis value can be determined precisely with a list like the public suffix\nlist (http://publicsuffix.org). Trying to approximate this by simply taking\nthe last label will not work well for effective TLDs such as "co.uk".', + example: 'co.uk', + }, + { + name: 'user.domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the directory the user is a member of.\n\nFor example, an LDAP or Active Directory domain name.', + }, + { + name: 'user.email', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'User email address.', + }, + { + name: 'user.full_name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ { - name: 'kibana.log.meta.statusCode', - type: 'alias', - path: 'http.response.status_code', - migration: true, + name: 'text', + type: 'text', + norms: false, + default_field: false, }, + ], + description: "User's full name, if available.", + example: 'Albert Einstein', + }, + { + name: 'user.group.domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the directory the group is a member of.\n\nFor example, an LDAP or Active Directory domain name.', + }, + { + name: 'user.group.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifier for the group on the system/platform.', + }, + { + name: 'user.group.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the group.', + }, + { + name: 'user.hash', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Unique user hash to correlate information for a user in anonymized\nform.\n\nUseful if `user.id` or `user.name` contain confidential information and cannot\nbe used.', + }, + { + name: 'user.id', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifiers of the user.', + }, + { + name: 'user.name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ { - name: 'kibana.log.meta.method', - type: 'alias', - path: 'http.request.method', - migration: true, + name: 'text', + type: 'text', + norms: false, + default_field: false, }, ], + description: 'Short name or login of the user.', + example: 'albert', }, ], }, - ], - }, - { - key: 'logstash', - title: 'logstash', - description: 'logstash Module', - fields: [ { - name: 'logstash', + name: 'service', + title: 'Service', + group: 2, + description: + 'The service fields describe the service for or from which the data\nwas collected.\n\nThese fields help you find and correlate logs for a specific service and version.', type: 'group', - description: '', fields: [ { - name: 'log', - title: 'Logstash', - type: 'group', - description: 'Fields from the Logstash logs.', - fields: [ - { - name: 'module', - type: 'keyword', - description: 'The module or class where the event originate.', - }, - { - name: 'thread', - type: 'keyword', - description: 'Information about the running thread where the log originate.', - multi_fields: [ - { - name: 'text', - type: 'text', - }, - ], - }, - { - name: 'log_event', - type: 'object', - description: 'key and value debugging information.', - }, - { - name: 'message', - type: 'alias', - path: 'message', - migration: true, - }, - { - name: 'level', - type: 'alias', - path: 'log.level', - migration: true, - }, - ], + name: 'ephemeral_id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Ephemeral identifier of this service (if one exists).\n\nThis id normally changes across restarts, but `service.id` does not.', + example: '8a4f500f', }, { - name: 'slowlog', - type: 'group', - description: 'slowlog', - fields: [ - { - name: 'module', - type: 'keyword', - description: 'The module or class where the event originate.', - }, - { - name: 'thread', - type: 'keyword', - description: 'Information about the running thread where the log originate.', - multi_fields: [ - { - name: 'text', - type: 'text', - }, - ], - }, - { - name: 'event', - type: 'keyword', - description: 'Raw dump of the original event', - multi_fields: [ - { - name: 'text', - type: 'text', - }, - ], - }, - { - name: 'plugin_name', - type: 'keyword', - description: 'Name of the plugin', - }, - { - name: 'plugin_type', - type: 'keyword', - description: 'Type of the plugin: Inputs, Filters, Outputs or Codecs.', - }, - { - name: 'took_in_millis', - type: 'long', - description: 'Execution time for the plugin in milliseconds.', - }, - { - name: 'plugin_params', - type: 'keyword', - description: 'String value of the plugin configuration', - multi_fields: [ - { - name: 'text', - type: 'text', - }, - ], - }, - { - name: 'plugin_params_object', - type: 'object', - description: 'key -> value of the configuration used by the plugin.', - }, - { - name: 'level', - type: 'alias', - path: 'log.level', - migration: true, - }, - { - name: 'took_in_nanos', - type: 'alias', - path: 'event.duration', - migration: true, - }, - ], + name: 'id', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'Unique identifier of the running service. If the service is comprised\nof many nodes, the `service.id` should be the same for all nodes.\n\nThis id should uniquely identify the service. This makes it possible to correlate\nlogs and metrics for one specific service, no matter which particular node\nemitted the event.\n\nNote that if you need to see the events from one specific host of the service,\nyou should filter on that `host.name` or `host.id` instead.', + example: 'd37e5ebfe0ae6c4972dbe9f0174a1637bb8247f6', + }, + { + name: 'name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the service data is collected from.\n\nThe name of the service is normally user given. This allows for distributed\nservices that run on multiple hosts to correlate the related instances based\non the name.\n\nIn the case of Elasticsearch the `service.name` could contain the cluster\nname. For Beats the `service.name` is by default a copy of the `service.type`\nfield if no name is specified.', + example: 'elasticsearch-metrics', + }, + { + name: 'node.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of a service node.\n\nThis allows for two nodes of the same service running on the same host to\nbe differentiated. Therefore, `service.node.name` should typically be unique\nacross nodes of a given service.\n\nIn the case of Elasticsearch, the `service.node.name` could contain the unique\nnode name within the Elasticsearch cluster. In cases where the service does not\nhave the concept of a node name, the host name or container name can be used\nto distinguish running instances that make up this service. If those do not\nprovide uniqueness (e.g. multiple instances of the service running on the\nsame host) - the node name can be manually set.', + example: 'instance-0000000016', + }, + { + name: 'state', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Current state of the service.', + }, + { + name: 'type', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'The type of the service data is collected from.\n\nThe type can be used to group and correlate logs and metrics from one service\ntype.\n\nExample: If logs or metrics are collected from Elasticsearch, `service.type`\nwould be `elasticsearch`.', + example: 'elasticsearch', + }, + { + name: 'version', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'Version of the service the data was collected from.\n\nThis allows to look at a data set only for a specific version of a service.', + example: '3.2.4', }, ], }, - ], - }, - { - key: 'mongodb', - title: 'mongodb', - description: 'Module for parsing MongoDB log files.', - fields: [ { - name: 'mongodb', + name: 'source', + title: 'Source', + group: 2, + description: + 'Source fields describe details about the source of a packet/event.\n\nSource fields are usually populated in conjunction with destination fields.', type: 'group', - description: 'Fields from MongoDB logs.', fields: [ { - name: 'log', - type: 'group', - description: 'Contains fields from MongoDB logs.', - fields: [ - { - name: 'component', - description: 'Functional categorization of message', - example: 'COMMAND', - type: 'keyword', - }, - { - name: 'context', - description: 'Context of message', - example: 'initandlisten', - type: 'keyword', - }, - { - name: 'severity', - type: 'alias', - path: 'log.level', - migration: true, - }, + name: 'address', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Some event source addresses are defined ambiguously. The event\nwill sometimes list an IP, a domain or a unix socket. You should always store\nthe raw address in the `.address` field.\n\nThen it should be duplicated to `.ip` or `.domain`, depending on which one\nit is.', + }, + { + name: 'as.number', + level: 'extended', + type: 'long', + description: + 'Unique number allocated to the autonomous system. The autonomous\nsystem number (ASN) uniquely identifies each network on the Internet.', + example: 15169, + }, + { + name: 'as.organization.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ { - name: 'message', - type: 'alias', - path: 'message', - migration: true, + name: 'text', + type: 'text', + norms: false, + default_field: false, }, ], + description: 'Organization name.', + example: 'Google LLC', }, - ], - }, - ], - }, - { - key: 'mysql', - title: 'MySQL', - description: 'Module for parsing the MySQL log files.', - short_config: true, - fields: [ - { - name: 'mysql', - type: 'group', - description: 'Fields from the MySQL log files.', - fields: [ { - name: 'thread_id', + name: 'bytes', + level: 'core', type: 'long', - description: 'The connection or thread ID for the query.', + format: 'bytes', + description: 'Bytes sent from the source to the destination.', + example: 184, }, { - name: 'error', - type: 'group', - description: 'Contains fields from the MySQL error logs.', - fields: [ - { - name: 'thread_id', - type: 'alias', - path: 'mysql.thread_id', - migration: true, - }, - { - name: 'level', - type: 'alias', - path: 'log.level', - migration: true, - }, + name: 'domain', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Source domain.', + }, + { + name: 'geo.city_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'City name.', + example: 'Montreal', + }, + { + name: 'geo.continent_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the continent.', + example: 'North America', + }, + { + name: 'geo.country_iso_code', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Country ISO code.', + example: 'CA', + }, + { + name: 'geo.country_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Country name.', + example: 'Canada', + }, + { + name: 'geo.location', + level: 'core', + type: 'geo_point', + description: 'Longitude and latitude.', + example: '{ "lon": -73.614830, "lat": 45.505918 }', + }, + { + name: 'geo.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'User-defined description of a location, at the level of granularity\nthey care about.\n\nCould be the name of their data centers, the floor number, if this describes\na local physical entity, city names.\n\nNot typically used in automated geolocation.', + example: 'boston-dc', + }, + { + name: 'geo.region_iso_code', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Region ISO code.', + example: 'CA-QC', + }, + { + name: 'geo.region_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Region name.', + example: 'Quebec', + }, + { + name: 'ip', + level: 'core', + type: 'ip', + description: + 'IP address of the source.\n\nCan be one or multiple IPv4 or IPv6 addresses.', + }, + { + name: 'mac', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'MAC address of the source.', + }, + { + name: 'nat.ip', + level: 'extended', + type: 'ip', + description: + 'Translated ip of source based NAT sessions (e.g. internal client\nto internet)\n\nTypically connections traversing load balancers, firewalls, or routers.', + }, + { + name: 'nat.port', + level: 'extended', + type: 'long', + format: 'string', + description: + 'Translated port of source based NAT sessions. (e.g. internal client\nto internet)\n\nTypically used with load balancers, firewalls, or routers.', + }, + { + name: 'packets', + level: 'core', + type: 'long', + description: 'Packets sent from the source to the destination.', + example: 12, + }, + { + name: 'port', + level: 'core', + type: 'long', + format: 'string', + description: 'Port of the source.', + }, + { + name: 'registered_domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The highest registered source domain, stripped of the subdomain.\n\nFor example, the registered domain for "foo.google.com" is "google.com".\n\nThis value can be determined precisely with a list like the public suffix\nlist (http://publicsuffix.org). Trying to approximate this by simply taking\nthe last two labels will not work well for TLDs such as "co.uk".', + example: 'google.com', + }, + { + name: 'top_level_domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The effective top level domain (eTLD), also known as the domain\nsuffix, is the last part of the domain name. For example, the top level domain\nfor google.com is "com".\n\nThis value can be determined precisely with a list like the public suffix\nlist (http://publicsuffix.org). Trying to approximate this by simply taking\nthe last label will not work well for effective TLDs such as "co.uk".', + example: 'co.uk', + }, + { + name: 'user.domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the directory the user is a member of.\n\nFor example, an LDAP or Active Directory domain name.', + }, + { + name: 'user.email', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'User email address.', + }, + { + name: 'user.full_name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ { - name: 'message', - type: 'alias', - path: 'message', - migration: true, + name: 'text', + type: 'text', + norms: false, + default_field: false, }, ], + description: "User's full name, if available.", + example: 'Albert Einstein', }, { - name: 'slowlog', - type: 'group', - description: 'Contains fields from the MySQL slow logs.', - fields: [ - { - name: 'lock_time.sec', - type: 'float', - description: - 'The amount of time the query waited for the lock to be available. The value is in seconds, as a floating point number.', - }, - { - name: 'rows_sent', - type: 'long', - description: 'The number of rows returned by the query.', - }, - { - name: 'rows_examined', - type: 'long', - description: 'The number of rows scanned by the query.', - }, - { - name: 'rows_affected', - type: 'long', - description: 'The number of rows modified by the query.', - }, - { - name: 'bytes_sent', - type: 'long', - format: 'bytes', - description: 'The size of the query result.', - }, - { - name: 'query', - description: 'The slow query.', - }, - { - name: 'id', - type: 'alias', - path: 'mysql.thread_id', - migration: true, - }, - { - name: 'schema', - type: 'keyword', - description: 'The schema where the slow query was executed.', - }, - { - name: 'current_user', - type: 'keyword', - description: - 'Current authenticated user, used to determine access privileges. Can differ from the value for user.', - }, - { - name: 'last_errno', - type: 'keyword', - description: 'Last SQL error seen.', - }, - { - name: 'killed', - type: 'keyword', - description: 'Code of the reason if the query was killed.', - }, - { - name: 'query_cache_hit', - type: 'boolean', - description: 'Whether the query cache was hit.', - }, - { - name: 'tmp_table', - type: 'boolean', - description: 'Whether a temporary table was used to resolve the query.', - }, - { - name: 'tmp_table_on_disk', - type: 'boolean', - description: 'Whether the query needed temporary tables on disk.', - }, - { - name: 'tmp_tables', - type: 'long', - description: 'Number of temporary tables created for this query', - }, - { - name: 'tmp_disk_tables', - type: 'long', - description: 'Number of temporary tables created on disk for this query.', - }, - { - name: 'tmp_table_sizes', - type: 'long', - format: 'bytes', - description: 'Size of temporary tables created for this query.', - }, - { - name: 'filesort', - type: 'boolean', - description: 'Whether filesort optimization was used.', - }, - { - name: 'filesort_on_disk', - type: 'boolean', - description: - 'Whether filesort optimization was used and it needed temporary tables on disk.', - }, - { - name: 'priority_queue', - type: 'boolean', - description: 'Whether a priority queue was used for filesort.', - }, - { - name: 'full_scan', - type: 'boolean', - description: 'Whether a full table scan was needed for the slow query.', - }, - { - name: 'full_join', - type: 'boolean', - description: - 'Whether a full join was needed for the slow query (no indexes were used for joins).', - }, - { - name: 'merge_passes', - type: 'long', - description: 'Number of merge passes executed for the query.', - }, - { - name: 'log_slow_rate_type', - type: 'keyword', - description: - 'Type of slow log rate limit, it can be `session` if the rate limit is applied per session, or `query` if it applies per query.', - }, - { - name: 'log_slow_rate_limit', - type: 'keyword', - description: - 'Slow log rate limit, a value of 100 means that one in a hundred queries or sessions are being logged.', - }, - { - name: 'innodb', - type: 'group', - description: 'Contains fields relative to InnoDB engine', - fields: [ - { - name: 'trx_id', - type: 'keyword', - description: 'Transaction ID', - }, - { - name: 'io_r_ops', - type: 'long', - description: 'Number of page read operations.', - }, - { - name: 'io_r_bytes', - type: 'long', - format: 'bytes', - description: 'Bytes read during page read operations.', - }, - { - name: 'io_r_wait.sec', - type: 'long', - description: 'How long it took to read all needed data from storage.', - }, - { - name: 'rec_lock_wait.sec', - type: 'long', - description: 'How long the query waited for locks.', - }, - { - name: 'queue_wait.sec', - type: 'long', - description: - 'How long the query waited to enter the InnoDB queue and to be executed once in the queue.', - }, - { - name: 'pages_distinct', - type: 'long', - description: 'Approximated count of pages accessed to execute the query.', - }, - ], - }, - { - name: 'user', - type: 'alias', - path: 'user.name', - migration: true, - }, - { - name: 'host', - type: 'alias', - path: 'source.domain', - migration: true, - }, + name: 'user.group.domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the directory the group is a member of.\n\nFor example, an LDAP or Active Directory domain name.', + }, + { + name: 'user.group.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifier for the group on the system/platform.', + }, + { + name: 'user.group.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the group.', + }, + { + name: 'user.hash', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Unique user hash to correlate information for a user in anonymized\nform.\n\nUseful if `user.id` or `user.name` contain confidential information and cannot\nbe used.', + }, + { + name: 'user.id', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifiers of the user.', + }, + { + name: 'user.name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ { - name: 'ip', - type: 'alias', - path: 'source.ip', - migration: true, + name: 'text', + type: 'text', + norms: false, + default_field: false, }, ], + description: 'Short name or login of the user.', + example: 'albert', }, ], }, - ], - }, - { - key: 'nats', - title: 'nats', - description: 'Module for parsing NATS log files.', - release: 'beta', - fields: [ { - name: 'nats', + name: 'threat', + title: 'Threat', + group: 2, + description: + 'Fields to classify events and alerts according to a threat taxonomy\nsuch as the Mitre ATT&CK framework.\n\nThese fields are for users to classify alerts from all of their sources (e.g.\nIDS, NGFW, etc.) within a common taxonomy. The threat.tactic.* are meant to\ncapture the high level category of the threat (e.g. "impact"). The threat.technique.*\nfields are meant to capture which kind of approach is used by this detected\nthreat, to accomplish the goal (e.g. "endpoint denial of service").', type: 'group', - description: 'Fields from NATS logs.', fields: [ { - name: 'log', - type: 'group', - description: 'Nats log files', - release: 'beta', + name: 'framework', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the threat framework used to further categorize and classify\nthe tactic and technique of the reported threat. Framework classification\ncan be provided by detecting systems, evaluated at ingest time, or retrospectively\ntagged to events.', + example: 'MITRE ATT&CK', + }, + { + name: 'tactic.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The id of tactic used by this threat. You can use the Mitre ATT&CK\nMatrix Tactic categorization, for example. (ex. https://attack.mitre.org/tactics/TA0040/\n)', + example: 'TA0040', + }, + { + name: 'tactic.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the type of tactic used by this threat. You can use the\nMitre ATT&CK Matrix Tactic categorization, for example. (ex. https://attack.mitre.org/tactics/TA0040/\n)', + example: 'impact', + }, + { + name: 'tactic.reference', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The reference url of tactic used by this threat. You can use the\nMitre ATT&CK Matrix Tactic categorization, for example. (ex. https://attack.mitre.org/tactics/TA0040/\n)', + example: 'https://attack.mitre.org/tactics/TA0040/', + }, + { + name: 'technique.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The id of technique used by this tactic. You can use the Mitre\nATT&CK Matrix Tactic categorization, for example. (ex. https://attack.mitre.org/techniques/T1499/\n)', + example: 'T1499', + }, + { + name: 'technique.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: + 'The name of technique used by this tactic. You can use the Mitre\nATT&CK Matrix Tactic categorization, for example. (ex. https://attack.mitre.org/techniques/T1499/\n)', + example: 'endpoint denial of service', + }, + { + name: 'technique.reference', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The reference url of technique used by this tactic. You can use\nthe Mitre ATT&CK Matrix Tactic categorization, for example. (ex. https://attack.mitre.org/techniques/T1499/\n)', + example: 'https://attack.mitre.org/techniques/T1499/', }, ], }, - ], - }, - { - key: 'nginx', - title: 'Nginx', - description: 'Module for parsing the Nginx log files.', - short_config: true, - fields: [ { - name: 'nginx', + name: 'tls', + title: 'TLS', + group: 2, + description: + 'Fields related to a TLS connection. These fields focus on the TLS\nprotocol itself and intentionally avoids in-depth analysis of the related x.509\ncertificate files.', type: 'group', - description: 'Fields from the Nginx log files.', fields: [ { - name: 'access', - type: 'group', - description: 'Contains fields for the Nginx access logs.', - fields: [ - { - name: 'remote_ip_list', - type: 'array', - description: - 'An array of remote IP addresses. It is a list because it is common to include, besides the client IP address, IP addresses from headers like `X-Forwarded-For`. Real source IP is restored to `source.ip`.', - }, - { - name: 'body_sent.bytes', - type: 'alias', - path: 'http.response.body.bytes', - migration: true, - }, - { - name: 'remote_ip', - type: 'alias', - path: 'source.ip', - migration: true, - }, - { - name: 'user_name', - type: 'alias', - path: 'user.name', - migration: true, - }, - { - name: 'method', - type: 'alias', - path: 'http.request.method', - migration: true, - }, - { - name: 'url', - type: 'alias', - path: 'url.original', - migration: true, - }, - { - name: 'http_version', - type: 'alias', - path: 'http.version', - migration: true, - }, - { - name: 'response_code', - type: 'alias', - path: 'http.response.status_code', - migration: true, - }, - { - name: 'referrer', - type: 'alias', - path: 'http.request.referrer', - migration: true, - }, - { - name: 'agent', - type: 'alias', - path: 'user_agent.original', - migration: true, - }, - { - name: 'user_agent', - type: 'group', - fields: [ - { - name: 'device', - type: 'alias', - path: 'user_agent.device.name', - migration: true, - }, - { - name: 'name', - type: 'alias', - path: 'user_agent.name', - migration: true, - }, - { - name: 'os', - type: 'alias', - path: 'user_agent.os.full_name', - migration: true, - }, - { - name: 'os_name', - type: 'alias', - path: 'user_agent.os.name', - migration: true, - }, - { - name: 'original', - type: 'alias', - path: 'user_agent.original', - migration: true, - }, - ], - }, - { - name: 'geoip', - type: 'group', - fields: [ - { - name: 'continent_name', - type: 'alias', - path: 'source.geo.continent_name', - migration: true, - }, - { - name: 'country_iso_code', - type: 'alias', - path: 'source.geo.country_iso_code', - migration: true, - }, - { - name: 'location', - type: 'alias', - path: 'source.geo.location', - migration: true, - }, - { - name: 'region_name', - type: 'alias', - path: 'source.geo.region_name', - migration: true, - }, - { - name: 'city_name', - type: 'alias', - path: 'source.geo.city_name', - migration: true, - }, - { - name: 'region_iso_code', - type: 'alias', - path: 'source.geo.region_iso_code', - migration: true, - }, - ], - }, - ], + name: 'cipher', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'String indicating the cipher used during the current connection.', + example: 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256', + default_field: false, }, { - name: 'error', - type: 'group', - description: 'Contains fields for the Nginx error logs.', - fields: [ - { - name: 'connection_id', - type: 'long', - description: 'Connection identifier.', - }, - { - name: 'level', - type: 'alias', - path: 'log.level', - migration: true, - }, - { - name: 'pid', - type: 'alias', - path: 'process.pid', - migration: true, - }, - { - name: 'tid', - type: 'alias', - path: 'process.thread.id', - migration: true, - }, - { - name: 'message', - type: 'alias', - path: 'message', - migration: true, - }, - ], + name: 'client.certificate', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'PEM-encoded stand-alone certificate offered by the client. This\nis usually mutually-exclusive of `client.certificate_chain` since this value\nalso exists in that list.', + example: 'MII...', + default_field: false, }, - ], - }, - ], - }, - { - key: 'osquery', - title: 'Osquery', - description: 'Fields exported by the `osquery` module', - fields: [ - { - name: 'osquery', - type: 'group', - description: '', - fields: [ { - name: 'result', - type: 'group', - description: 'Common fields exported by the result metricset.', - fields: [ - { - name: 'name', - type: 'keyword', - description: 'The name of the query that generated this event.', - }, - { - name: 'action', - type: 'keyword', - description: - 'For incremental data, marks whether the entry was added or removed. It can be one of "added", "removed", or "snapshot".', - }, - { - name: 'host_identifier', - type: 'keyword', - description: - 'The identifier for the host on which the osquery agent is running. Normally the hostname.', - }, - { - name: 'unix_time', - type: 'long', - description: - 'Unix timestamp of the event, in seconds since the epoch. Used for computing the `@timestamp` column.', - }, - { - name: 'calendar_time', - type: 'keyword', - description: - 'String representation of the collection time, as formatted by osquery.', - }, - ], + name: 'client.certificate_chain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Array of PEM-encoded certificates that make up the certificate\nchain offered by the client. This is usually mutually-exclusive of `client.certificate`\nsince that value should be the first certificate in the chain.', + example: ['MII...', 'MII...'], + default_field: false, }, - ], - }, - ], - }, - { - key: 'postgresql', - title: 'PostgreSQL', - description: 'Module for parsing the PostgreSQL log files.', - short_config: true, - fields: [ - { - name: 'postgresql', - type: 'group', - description: 'Fields from PostgreSQL logs.', - fields: [ { - name: 'log', - type: 'group', - description: 'Fields from the PostgreSQL log files.', - fields: [ - { - name: 'timestamp', - description: 'The timestamp from the log line.', - }, - { - name: 'core_id', - type: 'long', - description: 'Core id', - }, - { - name: 'database', - example: 'mydb', - description: 'Name of database', - }, - { - name: 'query', - example: 'SELECT * FROM users;', - description: 'Query statement.', - }, - { - name: 'timezone', - type: 'alias', - path: 'event.timezone', - migration: true, - }, - { - name: 'thread_id', - type: 'alias', - path: 'process.pid', - migration: true, - }, - { - name: 'user', - type: 'alias', - path: 'user.name', - migration: true, - }, - { - name: 'level', - type: 'alias', - path: 'log.level', - migration: true, - }, - { - name: 'message', - type: 'alias', - path: 'message', - migration: true, - }, + name: 'client.hash.md5', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Certificate fingerprint using the MD5 digest of DER-encoded version\nof certificate offered by the client. For consistency with other hash values,\nthis value should be formatted as an uppercase hash.', + example: '0F76C7F2C55BFD7D8E8B8F4BFBF0C9EC', + default_field: false, + }, + { + name: 'client.hash.sha1', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Certificate fingerprint using the SHA1 digest of DER-encoded version\nof certificate offered by the client. For consistency with other hash values,\nthis value should be formatted as an uppercase hash.', + example: '9E393D93138888D288266C2D915214D1D1CCEB2A', + default_field: false, + }, + { + name: 'client.hash.sha256', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Certificate fingerprint using the SHA256 digest of DER-encoded\nversion of certificate offered by the client. For consistency with other hash\nvalues, this value should be formatted as an uppercase hash.', + example: '0687F666A054EF17A08E2F2162EAB4CBC0D265E1D7875BE74BF3C712CA92DAF0', + default_field: false, + }, + { + name: 'client.issuer', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Distinguished name of subject of the issuer of the x.509 certificate\npresented by the client.', + example: 'CN=MyDomain Root CA, OU=Infrastructure Team, DC=mydomain, DC=com', + default_field: false, + }, + { + name: 'client.ja3', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'A hash that identifies clients based on how they perform an SSL/TLS\nhandshake.', + example: 'd4e5b18d6b55c71272893221c96ba240', + default_field: false, + }, + { + name: 'client.not_after', + level: 'extended', + type: 'date', + description: + 'Date/Time indicating when client certificate is no longer considered\nvalid.', + example: '2021-01-01T00:00:00.000Z', + default_field: false, + }, + { + name: 'client.not_before', + level: 'extended', + type: 'date', + description: 'Date/Time indicating when client certificate is first considered\nvalid.', + example: '1970-01-01T00:00:00.000Z', + default_field: false, + }, + { + name: 'client.server_name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Also called an SNI, this tells the server which hostname to which\nthe client is attempting to connect. When this value is available, it should\nget copied to `destination.domain`.', + example: 'www.elastic.co', + default_field: false, + }, + { + name: 'client.subject', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Distinguished name of subject of the x.509 certificate presented\nby the client.', + example: 'CN=myclient, OU=Documentation Team, DC=mydomain, DC=com', + default_field: false, + }, + { + name: 'client.supported_ciphers', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Array of ciphers offered by the client during the client hello.', + example: [ + 'TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384', + 'TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384', + '...', ], + default_field: false, + }, + { + name: 'curve', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'String indicating the curve used for the given cipher, when applicable.', + example: 'secp256r1', + default_field: false, + }, + { + name: 'established', + level: 'extended', + type: 'boolean', + description: + 'Boolean flag indicating if the TLS negotiation was successful and\ntransitioned to an encrypted tunnel.', + default_field: false, + }, + { + name: 'next_protocol', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'String indicating the protocol being tunneled. Per the values in\nthe IANA registry (https://www.iana.org/assignments/tls-extensiontype-values/tls-extensiontype-values.xhtml#alpn-protocol-ids),\nthis string should be lower case.', + example: 'http/1.1', + default_field: false, + }, + { + name: 'resumed', + level: 'extended', + type: 'boolean', + description: + 'Boolean flag indicating if this TLS connection was resumed from\nan existing TLS negotiation.', + default_field: false, + }, + { + name: 'server.certificate', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'PEM-encoded stand-alone certificate offered by the server. This\nis usually mutually-exclusive of `server.certificate_chain` since this value\nalso exists in that list.', + example: 'MII...', + default_field: false, + }, + { + name: 'server.certificate_chain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Array of PEM-encoded certificates that make up the certificate\nchain offered by the server. This is usually mutually-exclusive of `server.certificate`\nsince that value should be the first certificate in the chain.', + example: ['MII...', 'MII...'], + default_field: false, + }, + { + name: 'server.hash.md5', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Certificate fingerprint using the MD5 digest of DER-encoded version\nof certificate offered by the server. For consistency with other hash values,\nthis value should be formatted as an uppercase hash.', + example: '0F76C7F2C55BFD7D8E8B8F4BFBF0C9EC', + default_field: false, + }, + { + name: 'server.hash.sha1', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Certificate fingerprint using the SHA1 digest of DER-encoded version\nof certificate offered by the server. For consistency with other hash values,\nthis value should be formatted as an uppercase hash.', + example: '9E393D93138888D288266C2D915214D1D1CCEB2A', + default_field: false, + }, + { + name: 'server.hash.sha256', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Certificate fingerprint using the SHA256 digest of DER-encoded\nversion of certificate offered by the server. For consistency with other hash\nvalues, this value should be formatted as an uppercase hash.', + example: '0687F666A054EF17A08E2F2162EAB4CBC0D265E1D7875BE74BF3C712CA92DAF0', + default_field: false, + }, + { + name: 'server.issuer', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Subject of the issuer of the x.509 certificate presented by the\nserver.', + example: 'CN=MyDomain Root CA, OU=Infrastructure Team, DC=mydomain, DC=com', + default_field: false, + }, + { + name: 'server.ja3s', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'A hash that identifies servers based on how they perform an SSL/TLS\nhandshake.', + example: '394441ab65754e2207b1e1b457b3641d', + default_field: false, + }, + { + name: 'server.not_after', + level: 'extended', + type: 'date', + description: + 'Timestamp indicating when server certificate is no longer considered\nvalid.', + example: '2021-01-01T00:00:00.000Z', + default_field: false, + }, + { + name: 'server.not_before', + level: 'extended', + type: 'date', + description: 'Timestamp indicating when server certificate is first considered\nvalid.', + example: '1970-01-01T00:00:00.000Z', + default_field: false, + }, + { + name: 'server.subject', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Subject of the x.509 certificate presented by the server.', + example: 'CN=www.mydomain.com, OU=Infrastructure Team, DC=mydomain, DC=com', + default_field: false, + }, + { + name: 'version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Numeric part of the version parsed from the original string.', + example: '1.2', + default_field: false, + }, + { + name: 'version_protocol', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Normalized lowercase protocol name parsed from original string.', + example: 'tls', + default_field: false, }, ], }, - ], - }, - { - key: 'redis', - title: 'Redis', - description: 'Redis Module', - fields: [ { - name: 'redis', + name: 'tracing', + title: 'Tracing', + group: 2, + description: + 'Distributed tracing makes it possible to analyze performance throughout\na microservice architecture all in one view. This is accomplished by tracing\nall of the requests - from the initial web request in the front-end service\n- to queries made through multiple back-end services.', type: 'group', - description: '', fields: [ { - name: 'log', - type: 'group', - description: 'Redis log files', - fields: [ - { - name: 'role', - type: 'keyword', - description: - 'The role of the Redis instance. Can be one of `master`, `slave`, `child` (for RDF/AOF writing child), or `sentinel`.', - }, - { - name: 'pid', - type: 'alias', - path: 'process.pid', - migration: true, - }, - { - name: 'level', - type: 'alias', - path: 'log.level', - migration: true, - }, - { - name: 'message', - type: 'alias', - path: 'message', - migration: true, - }, - ], + name: 'trace.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Unique identifier of the trace.\n\nA trace groups multiple events like transactions that belong together. For\nexample, a user request handled by multiple inter-connected services.', + example: '4bf92f3577b34da6a3ce929d0e0e4736', }, { - name: 'slowlog', - type: 'group', - description: 'Slow logs are retrieved from Redis via a network connection.', - fields: [ - { - name: 'cmd', - type: 'keyword', - description: 'The command executed.', - }, - { - name: 'duration.us', - type: 'long', - description: 'How long it took to execute the command in microseconds.', - }, - { - name: 'id', - type: 'long', - description: 'The ID of the query.', - }, - { - name: 'key', - type: 'keyword', - description: 'The key on which the command was executed.', - }, - { - name: 'args', - type: 'keyword', - description: 'The arguments with which the command was called.', - }, - ], + name: 'transaction.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Unique identifier of the transaction.\n\nA transaction is the highest level of work measured within a service, such\nas a request to a server.', + example: '00f067aa0ba902b7', }, ], }, - ], - }, - { - key: 'santa', - title: 'Google Santa', - description: 'Santa Module', - fields: [ { - name: 'santa', + name: 'url', + title: 'URL', + group: 2, + description: + 'URL fields provide support for complete or partial URLs, and supports\nthe breaking down into scheme, domain, path, and so on.', type: 'group', - description: '', fields: [ { - name: 'action', + name: 'domain', + level: 'extended', type: 'keyword', - example: 'EXEC', - description: 'Action', + ignore_above: 1024, + description: + 'Domain of the url, such as "www.elastic.co".\n\nIn some cases a URL may refer to an IP and/or port directly, without a domain\nname. In this case, the IP address would go to the `domain` field.', + example: 'www.elastic.co', }, { - name: 'decision', + name: 'extension', + level: 'extended', type: 'keyword', - example: 'ALLOW', - description: 'Decision that santad took.', + ignore_above: 1024, + description: + 'The field contains the file extension from the original request\nurl.\n\nThe file extension is only set if it exists, as not every url has a file extension.\n\nThe leading period must not be included. For example, the value must be "png",\nnot ".png".', + example: 'png', }, { - name: 'reason', + name: 'fragment', + level: 'extended', type: 'keyword', - example: 'CERT', - description: 'Reason for the decsision.', + ignore_above: 1024, + description: + 'Portion of the url after the `#`, such as "top".\n\nThe `#` is not part of the fragment.', }, { - name: 'mode', + name: 'full', + level: 'extended', type: 'keyword', - example: 'M', - description: 'Operating mode of Santa.', - }, - { - name: 'disk', - type: 'group', - description: 'Fields for DISKAPPEAR actions.', - fields: [ + ignore_above: 1024, + multi_fields: [ { - name: 'volume', - description: 'The volume name.', + name: 'text', + type: 'text', + norms: false, + default_field: false, }, + ], + description: + 'If full URLs are important to your use case, they should be stored\nin `url.full`, whether this field is reconstructed or present in the event\nsource.', + example: 'https://www.elastic.co:443/search?q=elasticsearch#top', + }, + { + name: 'original', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ { - name: 'bus', - description: 'The disk bus protocol.', + name: 'text', + type: 'text', + norms: false, + default_field: false, }, - { - name: 'serial', - description: 'The disk serial number.', + ], + description: + 'Unmodified original url as seen in the event source.\n\nNote that in network monitoring, the observed URL may be a full URL, whereas\nin access logs, the URL is often just represented as a path.\n\nThis field is meant to represent the URL as it was observed, complete or not.', + example: + 'https://www.elastic.co:443/search?q=elasticsearch#top or /search?q=elasticsearch', + }, + { + name: 'password', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Password of the request.', + }, + { + name: 'path', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Path of the request, such as "/search".', + }, + { + name: 'port', + level: 'extended', + type: 'long', + format: 'string', + description: 'Port of the request, such as 443.', + example: 443, + }, + { + name: 'query', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The query field describes the query string of the request, such\nas "q=elasticsearch".\n\nThe `?` is excluded from the query string. If a URL contains no `?`, there\nis no query field. If there is a `?` but no query, the query field exists\nwith an empty string. The `exists` query can be used to differentiate between\nthe two cases.', + }, + { + name: 'registered_domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The highest registered url domain, stripped of the subdomain.\n\nFor example, the registered domain for "foo.google.com" is "google.com".\n\nThis value can be determined precisely with a list like the public suffix\nlist (http://publicsuffix.org). Trying to approximate this by simply taking\nthe last two labels will not work well for TLDs such as "co.uk".', + example: 'google.com', + }, + { + name: 'scheme', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Scheme of the request, such as "https".\n\nNote: The `:` is not part of the scheme.', + example: 'https', + }, + { + name: 'top_level_domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The effective top level domain (eTLD), also known as the domain\nsuffix, is the last part of the domain name. For example, the top level domain\nfor google.com is "com".\n\nThis value can be determined precisely with a list like the public suffix\nlist (http://publicsuffix.org). Trying to approximate this by simply taking\nthe last label will not work well for effective TLDs such as "co.uk".', + example: 'co.uk', + }, + { + name: 'username', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Username of the request.', + }, + ], + }, + { + name: 'user', + title: 'User', + group: 2, + description: + 'The user fields describe information about the user that is relevant\nto the event.\n\nFields can have one entry or multiple entries. If a user has more than one id,\nprovide an array that includes all of them.', + type: 'group', + fields: [ + { + name: 'domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the directory the user is a member of.\n\nFor example, an LDAP or Active Directory domain name.', + }, + { + name: 'email', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'User email address.', + }, + { + name: 'full_name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, }, + ], + description: "User's full name, if available.", + example: 'Albert Einstein', + }, + { + name: 'group.domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the directory the group is a member of.\n\nFor example, an LDAP or Active Directory domain name.', + }, + { + name: 'group.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifier for the group on the system/platform.', + }, + { + name: 'group.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the group.', + }, + { + name: 'hash', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Unique user hash to correlate information for a user in anonymized\nform.\n\nUseful if `user.id` or `user.name` contain confidential information and cannot\nbe used.', + }, + { + name: 'id', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifiers of the user.', + }, + { + name: 'name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ { - name: 'bsdname', - example: 'disk1s3', - description: 'The disk BSD name.', + name: 'text', + type: 'text', + norms: false, + default_field: false, }, + ], + description: 'Short name or login of the user.', + example: 'albert', + }, + ], + }, + { + name: 'user_agent', + title: 'User agent', + group: 2, + description: + 'The user_agent fields normally come from a browser request.\n\nThey often show up in web service logs coming from the parsed user agent string.', + type: 'group', + fields: [ + { + name: 'device.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the device.', + example: 'iPhone', + }, + { + name: 'name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the user agent.', + example: 'Safari', + }, + { + name: 'original', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ { - name: 'model', - example: 'APPLE SSD SM0512L', - description: 'The disk model.', + name: 'text', + type: 'text', + norms: false, }, + ], + description: 'Unparsed user_agent string.', + example: + 'Mozilla/5.0 (iPhone; CPU iPhone OS 12_1 like Mac OS X) AppleWebKit/605.1.15\n(KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1', + }, + { + name: 'os.family', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'OS family (such as redhat, debian, freebsd, windows).', + example: 'debian', + }, + { + name: 'os.full', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ { - name: 'fs', - example: 'apfs', - description: 'The disk volume kind (filesystem type).', + name: 'text', + type: 'text', + norms: false, + default_field: false, }, + ], + description: 'Operating system name, including the version or code name.', + example: 'Mac OS Mojave', + }, + { + name: 'os.kernel', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Operating system kernel version as a raw string.', + example: '4.4.0-112-generic', + }, + { + name: 'os.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ { - name: 'mount', - description: 'The disk volume path.', + name: 'text', + type: 'text', + norms: false, + default_field: false, }, ], + description: 'Operating system name, without the version.', + example: 'Mac OS X', + }, + { + name: 'os.platform', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Operating system platform (such centos, ubuntu, windows).', + example: 'darwin', + }, + { + name: 'os.version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Operating system version as a raw string.', + example: '10.14.1', + }, + { + name: 'version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Version of the user agent.', + example: 12, }, ], }, { - name: 'certificate.common_name', - type: 'keyword', - description: 'Common name from code signing certificate.', - }, - { - name: 'certificate.sha256', - type: 'keyword', - description: 'SHA256 hash of code signing certificate.', - }, - { - name: 'hash.sha256', - type: 'keyword', - description: 'Hash of process executable.', + name: 'vlan', + title: 'VLAN', + group: 2, + description: + 'The VLAN fields are used to identify 802.1q tag(s) of a packet,\nas well as ingress and egress VLAN associations of an observer in relation to\na specific packet or connection.\n\nNetwork.vlan fields are used to record a single VLAN tag, or the outer tag in\nthe case of q-in-q encapsulations, for a packet or connection as observed, typically\nprovided by a network sensor (e.g. Zeek, Wireshark) passively reporting on traffic.\n\nNetwork.inner VLAN fields are used to report inner q-in-q 802.1q tags (multiple\n802.1q encapsulations) as observed, typically provided by a network sensor (e.g.\nZeek, Wireshark) passively reporting on traffic. Network.inner VLAN fields should\nonly be used in addition to network.vlan fields to indicate q-in-q tagging.\n\nObserver.ingress and observer.egress VLAN values are used to record observer\nspecific information when observer events contain discrete ingress and egress\nVLAN information, typically provided by firewalls, routers, or load balancers.', + type: 'group', + fields: [ + { + name: 'id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'VLAN ID as reported by the observer.', + example: 10, + default_field: false, + }, + { + name: 'name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Optional VLAN name as reported by the observer.', + example: 'outside', + default_field: false, + }, + ], }, - ], - }, - { - key: 'system', - title: 'System', - description: 'Module for parsing system log files.', - short_config: true, - fields: [ { - name: 'system', + name: 'vulnerability', + title: 'Vulnerability', + group: 2, + description: + 'The vulnerability fields describe information about a vulnerability\nthat is relevant to an event.', type: 'group', - description: 'Fields from the system log files.', fields: [ { - name: 'auth', - type: 'group', - description: 'Fields from the Linux authorization logs.', - fields: [ + name: 'category', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The type of system or architecture that the vulnerability affects.\nThese may be platform-specific (for example, Debian or SUSE) or general (for\nexample, Database or Firewall). For example (https://qualysguard.qualys.com/qwebhelp/fo_portal/knowledgebase/vulnerability_categories.htm[Qualys\nvulnerability categories])\n\nThis field must be an array.', + example: '["Firewall"]', + default_field: false, + }, + { + name: 'classification', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The classification of the vulnerability scoring system. For example\n(https://www.first.org/cvss/)', + example: 'CVSS', + default_field: false, + }, + { + name: 'description', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ { - name: 'timestamp', - type: 'alias', - path: '@timestamp', - migration: true, - }, - { - name: 'hostname', - type: 'alias', - path: 'host.hostname', - migration: true, - }, - { - name: 'program', - type: 'alias', - path: 'process.name', - migration: true, - }, - { - name: 'pid', - type: 'alias', - path: 'process.pid', - migration: true, - }, - { - name: 'message', - type: 'alias', - path: 'message', - migration: true, - }, - { - name: 'user', - type: 'alias', - path: 'user.name', - migration: true, - }, - { - name: 'ssh', - type: 'group', - fields: [ - { - name: 'method', - description: - 'The SSH authentication method. Can be one of "password" or "publickey".', - }, - { - name: 'signature', - description: 'The signature of the client public key.', - }, - { - name: 'dropped_ip', - type: 'ip', - description: - 'The client IP from SSH connections that are open and immediately dropped.', - }, - { - name: 'event', - type: 'alias', - path: 'event.action', - migration: true, - }, - { - name: 'ip', - type: 'alias', - path: 'source.ip', - migration: true, - }, - { - name: 'port', - type: 'alias', - path: 'source.port', - migration: true, - }, - { - name: 'geoip', - type: 'group', - fields: [ - { - name: 'continent_name', - type: 'alias', - path: 'source.geo.continent_name', - migration: true, - }, - { - name: 'country_iso_code', - type: 'alias', - path: 'source.geo.country_iso_code', - migration: true, - }, - { - name: 'location', - type: 'alias', - path: 'source.geo.location', - migration: true, - }, - { - name: 'region_name', - type: 'alias', - path: 'source.geo.region_name', - migration: true, - }, - { - name: 'city_name', - type: 'alias', - path: 'source.geo.city_name', - migration: true, - }, - { - name: 'region_iso_code', - type: 'alias', - path: 'source.geo.region_iso_code', - migration: true, - }, - ], - }, - ], - }, - { - name: 'sudo', - type: 'group', - description: 'Fields specific to events created by the `sudo` command.', - fields: [ - { - name: 'error', - example: 'user NOT in sudoers', - description: 'The error message in case the sudo command failed.', - }, - { - name: 'tty', - description: 'The TTY where the sudo command is executed.', - }, - { - name: 'pwd', - description: 'The current directory where the sudo command is executed.', - }, - { - name: 'user', - example: 'root', - description: 'The target user to which the sudo command is switching.', - }, - { - name: 'command', - description: 'The command executed via sudo.', - }, - ], - }, - { - name: 'useradd', - type: 'group', - description: 'Fields specific to events created by the `useradd` command.', - fields: [ - { - name: 'home', - description: 'The home folder for the new user.', - }, - { - name: 'shell', - description: 'The default shell for the new user.', - }, - { - name: 'name', - type: 'alias', - path: 'user.name', - migration: true, - }, - { - name: 'uid', - type: 'alias', - path: 'user.id', - migration: true, - }, - { - name: 'gid', - type: 'alias', - path: 'group.id', - migration: true, - }, - ], - }, - { - name: 'groupadd', - type: 'group', - description: 'Fields specific to events created by the `groupadd` command.', - fields: [ - { - name: 'name', - type: 'alias', - path: 'group.name', - migration: true, - }, - { - name: 'gid', - type: 'alias', - path: 'group.id', - migration: true, - }, - ], + name: 'text', + type: 'text', + norms: false, }, ], + description: + 'The description of the vulnerability that provides additional context\nof the vulnerability. For example (https://cve.mitre.org/about/faqs.html#cve_entry_descriptions_created[Common\nVulnerabilities and Exposure CVE description])', + example: 'In macOS before 2.12.6, there is a vulnerability in the RPC...', + default_field: false, }, { - name: 'syslog', - type: 'group', - description: 'Contains fields from the syslog system logs.', - fields: [ - { - name: 'timestamp', - type: 'alias', - path: '@timestamp', - migration: true, - }, - { - name: 'hostname', - type: 'alias', - path: 'host.hostname', - migration: true, - }, - { - name: 'program', - type: 'alias', - path: 'process.name', - migration: true, - }, - { - name: 'pid', - type: 'alias', - path: 'process.pid', - migration: true, - }, - { - name: 'message', - type: 'alias', - path: 'message', - migration: true, - }, - ], + name: 'enumeration', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The type of identifier used for this vulnerability. For example\n(https://cve.mitre.org/about/)', + example: 'CVE', + default_field: false, }, - ], - }, - ], - }, - { - key: 'traefik', - title: 'Traefik', - description: 'Module for parsing the Traefik log files.', - fields: [ - { - name: 'traefik', - type: 'group', - description: 'Fields from the Traefik log files.', - fields: [ { - name: 'access', - type: 'group', - description: 'Contains fields for the Traefik access logs.', - fields: [ - { - name: 'user_identifier', - type: 'keyword', - description: 'Is the RFC 1413 identity of the client', - }, - { - name: 'request_count', - type: 'long', - description: 'The number of requests', - }, - { - name: 'frontend_name', - type: 'keyword', - description: 'The name of the frontend used', - }, - { - name: 'backend_url', - type: 'keyword', - description: 'The url of the backend where request is forwarded', - }, - { - name: 'body_sent.bytes', - type: 'alias', - path: 'http.response.body.bytes', - migration: true, - }, - { - name: 'remote_ip', - type: 'alias', - path: 'source.address', - migration: true, - }, - { - name: 'user_name', - type: 'alias', - path: 'user.name', - migration: true, - }, - { - name: 'method', - type: 'alias', - path: 'http.request.method', - migration: true, - }, - { - name: 'url', - type: 'alias', - path: 'url.original', - migration: true, - }, - { - name: 'http_version', - type: 'alias', - path: 'http.version', - migration: true, - }, - { - name: 'response_code', - type: 'alias', - path: 'http.response.status_code', - migration: true, - }, - { - name: 'referrer', - type: 'alias', - path: 'http.request.referrer', - migration: true, - }, - { - name: 'agent', - type: 'alias', - path: 'user_agent.original', - migration: true, - }, - { - name: 'user_agent', - type: 'group', - fields: [ - { - name: 'device', - type: 'alias', - path: 'user_agent.device.name', - }, - { - name: 'name', - type: 'alias', - path: 'user_agent.name', - }, - { - name: 'os', - type: 'alias', - path: 'user_agent.os.full_name', - }, - { - name: 'os_name', - type: 'alias', - path: 'user_agent.os.name', - }, - { - name: 'original', - type: 'alias', - path: 'user_agent.original', - }, - ], - }, - { - name: 'geoip', - type: 'group', - fields: [ - { - name: 'continent_name', - type: 'alias', - path: 'source.geo.continent_name', - }, - { - name: 'country_iso_code', - type: 'alias', - path: 'source.geo.country_iso_code', - }, - { - name: 'location', - type: 'alias', - path: 'source.geo.location', - }, - { - name: 'region_name', - type: 'alias', - path: 'source.geo.region_name', - }, - { - name: 'city_name', - type: 'alias', - path: 'source.geo.city_name', - }, - { - name: 'region_iso_code', - type: 'alias', - path: 'source.geo.region_iso_code', - }, - ], - }, - ], - }, - ], - }, - ], - }, - { - key: 'iptables', - title: 'iptables', - description: 'Module for handling the iptables logs.', - fields: [ - { - name: 'iptables', - type: 'group', - description: 'Fields from the iptables logs.', - fields: [ - { - name: 'ether_type', - type: 'long', - description: 'Value of the ethernet type field identifying the network layer protocol.', - }, - { - name: 'flow_label', - type: 'integer', - description: 'IPv6 flow label.', - }, - { - name: 'fragment_flags', + name: 'id', + level: 'extended', type: 'keyword', - description: 'IP fragment flags. A combination of CE, DF and MF.', - }, - { - name: 'fragment_offset', - type: 'long', - description: 'Offset of the current IP fragment.', - }, - { - name: 'icmp', - type: 'group', - description: 'ICMP fields.', - fields: [ - { - name: 'code', - type: 'long', - description: 'ICMP code.', - }, - { - name: 'id', - type: 'long', - description: 'ICMP ID.', - }, - { - name: 'parameter', - type: 'long', - description: 'ICMP parameter.', - }, - { - name: 'redirect', - type: 'ip', - description: 'ICMP redirect address.', - }, - { - name: 'seq', - type: 'long', - description: 'ICMP sequence number.', - }, - { - name: 'type', - type: 'long', - description: 'ICMP type.', - }, - ], + ignore_above: 1024, + description: + 'The identification (ID) is the number portion of a vulnerability\nentry. It includes a unique identification number for the vulnerability. For\nexample (https://cve.mitre.org/about/faqs.html#what_is_cve_id)[Common Vulnerabilities\nand Exposure CVE ID]', + example: 'CVE-2019-00001', + default_field: false, }, { - name: 'id', - type: 'long', - description: 'Packet identifier.', + name: 'reference', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'A resource that provides additional information, context, and mitigations\nfor the identified vulnerability.', + example: 'https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-6111', + default_field: false, }, { - name: 'incomplete_bytes', - type: 'long', - description: 'Number of incomplete bytes.', + name: 'report_id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'The report or scan identification number.', + example: 20191018.0001, + default_field: false, }, { - name: 'input_device', + name: 'scanner.vendor', + level: 'extended', type: 'keyword', - description: 'Device that received the packet.', + ignore_above: 1024, + description: 'The name of the vulnerability scanner vendor.', + example: 'Tenable', + default_field: false, }, { - name: 'precedence_bits', - type: 'short', - description: 'IP precedence bits.', + name: 'score.base', + level: 'extended', + type: 'float', + description: + 'Scores can range from 0.0 to 10.0, with 10.0 being the most severe.\n\nBase scores cover an assessment for exploitability metrics (attack vector,\ncomplexity, privileges, and user interaction), impact metrics (confidentiality,\nintegrity, and availability), and scope. For example (https://www.first.org/cvss/specification-document)', + example: 5.5, + default_field: false, }, { - name: 'tos', - type: 'long', - description: 'IP Type of Service field.', + name: 'score.environmental', + level: 'extended', + type: 'float', + description: + 'Scores can range from 0.0 to 10.0, with 10.0 being the most severe.\n\nEnvironmental scores cover an assessment for any modified Base metrics, confidentiality,\nintegrity, and availability requirements. For example (https://www.first.org/cvss/specification-document)', + example: 5.5, + default_field: false, }, { - name: 'length', - type: 'long', - description: 'Packet length.', + name: 'score.temporal', + level: 'extended', + type: 'float', + description: + 'Scores can range from 0.0 to 10.0, with 10.0 being the most severe.\n\nTemporal scores cover an assessment for code maturity, remediation level,\nand confidence. For example (https://www.first.org/cvss/specification-document)', + default_field: false, }, { - name: 'output_device', + name: 'score.version', + level: 'extended', type: 'keyword', - description: 'Device that output the packet.', - }, - { - name: 'tcp', - type: 'group', - description: 'TCP fields.', - fields: [ - { - name: 'flags', - type: 'keyword', - description: 'TCP flags.', - }, - { - name: 'reserved_bits', - type: 'short', - description: 'TCP reserved bits.', - }, - { - name: 'seq', - type: 'long', - description: 'TCP sequence number.', - }, - { - name: 'ack', - type: 'long', - description: 'TCP Acknowledgment number.', - }, - { - name: 'window', - type: 'long', - description: 'Advertised TCP window size.', - }, - ], - }, - { - name: 'ttl', - type: 'integer', - description: 'Time To Live field.', + ignore_above: 1024, + description: + 'The National Vulnerability Database (NVD) provides qualitative\nseverity rankings of "Low", "Medium", and "High" for CVSS v2.0 base score\nranges in addition to the severity ratings for CVSS v3.0 as they are defined\nin the CVSS v3.0 specification.\n\nCVSS is owned and managed by FIRST.Org, Inc. (FIRST), a US-based non-profit\norganization, whose mission is to help computer security incident response\nteams across the world. For example (https://nvd.nist.gov/vuln-metrics/cvss)', + example: 2, + default_field: false, }, { - name: 'udp', - type: 'group', - description: 'UDP fields.', - fields: [ - { - name: 'length', - type: 'long', - description: 'Length of the UDP header and payload.', - }, - ], + name: 'severity', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The severity of the vulnerability can help with metrics and internal\nprioritization regarding remediation. For example (https://nvd.nist.gov/vuln-metrics/cvss)', + example: 'Critical', + default_field: false, }, - { - name: 'ubiquiti', - type: 'group', - description: 'Fields for Ubiquiti network devices.', - fields: [ - { - name: 'input_zone', - type: 'keyword', - description: 'Input zone.', - }, - { - name: 'output_zone', - type: 'keyword', - description: 'Output zone.', - }, - { - name: 'rule_number', - type: 'keyword', - description: 'The rule number within the rule set.', - }, - { - name: 'rule_set', - type: 'keyword', - description: 'The rule set name.', - }, - ], + ], + }, + ], + }, + { + key: 'beat', + anchor: 'beat-common', + title: 'Beat', + description: 'Contains common beat fields available in all event types.\n', + fields: [ + { + name: 'agent.hostname', + type: 'keyword', + description: 'Hostname of the agent.', + }, + { + name: 'beat.timezone', + type: 'alias', + path: 'event.timezone', + migration: true, + }, + { + name: 'fields', + type: 'object', + object_type: 'keyword', + description: 'Contains user configurable fields.\n', + }, + { + name: 'beat.name', + type: 'alias', + path: 'host.name', + migration: true, + }, + { + name: 'beat.hostname', + type: 'alias', + path: 'agent.hostname', + migration: true, + }, + { + name: 'timeseries.instance', + type: 'keyword', + description: 'Time series instance id', + }, + ], + }, + { + key: 'cloud', + title: 'Cloud provider metadata', + description: 'Metadata from cloud providers added by the add_cloud_metadata processor.\n', + fields: [ + { + name: 'cloud.project.id', + example: 'project-x', + description: 'Name of the project in Google Cloud.\n', + }, + { + name: 'cloud.image.id', + example: 'ami-abcd1234', + description: 'Image ID for the cloud instance.\n', + }, + { + name: 'meta.cloud.provider', + type: 'alias', + path: 'cloud.provider', + migration: true, + }, + { + name: 'meta.cloud.instance_id', + type: 'alias', + path: 'cloud.instance.id', + migration: true, + }, + { + name: 'meta.cloud.instance_name', + type: 'alias', + path: 'cloud.instance.name', + migration: true, + }, + { + name: 'meta.cloud.machine_type', + type: 'alias', + path: 'cloud.machine.type', + migration: true, + }, + { + name: 'meta.cloud.availability_zone', + type: 'alias', + path: 'cloud.availability_zone', + migration: true, + }, + { + name: 'meta.cloud.project_id', + type: 'alias', + path: 'cloud.project.id', + migration: true, + }, + { + name: 'meta.cloud.region', + type: 'alias', + path: 'cloud.region', + migration: true, + }, + ], + }, + { + key: 'docker', + title: 'Docker', + description: 'Docker stats collected from Docker.\n', + short_config: false, + anchor: 'docker-processor', + fields: [ + { + name: 'docker', + type: 'group', + fields: [ + { + name: 'container.id', + type: 'alias', + path: 'container.id', + migration: true, + }, + { + name: 'container.image', + type: 'alias', + path: 'container.image.name', + migration: true, + }, + { + name: 'container.name', + type: 'alias', + path: 'container.name', + migration: true, + }, + { + name: 'container.labels', + type: 'object', + object_type: 'keyword', + description: 'Image labels.\n', }, ], }, ], }, { - key: 'netflow-module', - title: 'NetFlow', - description: - 'Module for receiving NetFlow and IPFIX flow records over UDP. The module does not add fields beyond what the netflow input provides.', + key: 'host', + title: 'Host', + description: 'Info collected for the host machine.\n', + anchor: 'host-processor', + fields: [ + { + name: 'host', + type: 'group', + fields: [ + { + name: 'containerized', + type: 'boolean', + description: 'If the host is a container.\n', + }, + { + name: 'os.build', + type: 'keyword', + example: '18D109', + description: 'OS build information.\n', + }, + { + name: 'os.codename', + type: 'keyword', + example: 'stretch', + description: 'OS codename, if any.\n', + }, + ], + }, + ], }, { - key: 'suricata', - title: 'Suricata', - description: 'Module for handling the EVE JSON logs produced by Suricata.', + key: 'kubernetes', + title: 'Kubernetes', + description: 'Kubernetes metadata added by the kubernetes processor\n', + short_config: false, + anchor: 'kubernetes-processor', fields: [ { - name: 'suricata', + name: 'kubernetes', type: 'group', - description: 'Fields from the Suricata EVE log file.', fields: [ { - name: 'eve', - type: 'group', - description: 'Fields exported by the EVE JSON logs', - fields: [ - { - name: 'event_type', - type: 'keyword', - }, - { - name: 'app_proto_orig', - type: 'keyword', - }, - { - name: 'tcp', - type: 'group', - fields: [ - { - name: 'tcp_flags', - type: 'keyword', - }, - { - name: 'psh', - type: 'boolean', - }, - { - name: 'tcp_flags_tc', - type: 'keyword', - }, - { - name: 'ack', - type: 'boolean', - }, - { - name: 'syn', - type: 'boolean', - }, - { - name: 'state', - type: 'keyword', - }, - { - name: 'tcp_flags_ts', - type: 'keyword', - }, - { - name: 'rst', - type: 'boolean', - }, - { - name: 'fin', - type: 'boolean', - }, - ], - }, - { - name: 'fileinfo', - type: 'group', - fields: [ - { - name: 'sha1', - type: 'keyword', - }, - { - name: 'filename', - type: 'alias', - path: 'file.path', - }, - { - name: 'tx_id', - type: 'long', - }, - { - name: 'state', - type: 'keyword', - }, - { - name: 'stored', - type: 'boolean', - }, - { - name: 'gaps', - type: 'boolean', - }, - { - name: 'sha256', - type: 'keyword', - }, - { - name: 'md5', - type: 'keyword', - }, - { - name: 'size', - type: 'alias', - path: 'file.size', - }, - ], - }, - { - name: 'icmp_type', - type: 'long', - }, - { - name: 'dest_port', - type: 'alias', - path: 'destination.port', - }, - { - name: 'src_port', - type: 'alias', - path: 'source.port', - }, - { - name: 'proto', - type: 'alias', - path: 'network.transport', - }, - { - name: 'pcap_cnt', - type: 'long', - }, - { - name: 'src_ip', - type: 'alias', - path: 'source.ip', - }, - { - name: 'dns', - type: 'group', - fields: [ - { - name: 'type', - type: 'keyword', - }, - { - name: 'rrtype', - type: 'keyword', - }, - { - name: 'rrname', - type: 'keyword', - }, - { - name: 'rdata', - type: 'keyword', - }, - { - name: 'tx_id', - type: 'long', - }, - { - name: 'ttl', - type: 'long', - }, - { - name: 'rcode', - type: 'keyword', - }, - { - name: 'id', - type: 'long', - }, - ], - }, + name: 'pod.name', + type: 'keyword', + description: 'Kubernetes pod name\n', + }, + { + name: 'pod.uid', + type: 'keyword', + description: 'Kubernetes Pod UID\n', + }, + { + name: 'namespace', + type: 'keyword', + description: 'Kubernetes namespace\n', + }, + { + name: 'node.name', + type: 'keyword', + description: 'Kubernetes node name\n', + }, + { + name: 'labels.*', + type: 'object', + object_type: 'keyword', + object_type_mapping_type: '*', + description: 'Kubernetes labels map\n', + }, + { + name: 'annotations.*', + type: 'object', + object_type: 'keyword', + object_type_mapping_type: '*', + description: 'Kubernetes annotations map\n', + }, + { + name: 'replicaset.name', + type: 'keyword', + description: 'Kubernetes replicaset name\n', + }, + { + name: 'deployment.name', + type: 'keyword', + description: 'Kubernetes deployment name\n', + }, + { + name: 'statefulset.name', + type: 'keyword', + description: 'Kubernetes statefulset name\n', + }, + { + name: 'container.name', + type: 'keyword', + description: 'Kubernetes container name\n', + }, + { + name: 'container.image', + type: 'keyword', + description: 'Kubernetes container image\n', + }, + ], + }, + ], + }, + { + key: 'process', + title: 'Process', + description: 'Process metadata fields\n', + fields: [ + { + name: 'process', + type: 'group', + fields: [ + { + name: 'exe', + type: 'alias', + path: 'process.executable', + migration: true, + }, + ], + }, + ], + }, + { + key: 'jolokia-autodiscover', + title: 'Jolokia Discovery autodiscover provider', + description: 'Metadata from Jolokia Discovery added by the jolokia provider.\n', + fields: [ + { + name: 'jolokia.agent.version', + type: 'keyword', + description: 'Version number of jolokia agent.\n', + }, + { + name: 'jolokia.agent.id', + type: 'keyword', + description: + 'Each agent has a unique id which can be either provided during startup of the agent in form of a configuration parameter or being autodetected. If autodected, the id has several parts: The IP, the process id, hashcode of the agent and its type.\n', + }, + { + name: 'jolokia.server.product', + type: 'keyword', + description: 'The container product if detected.\n', + }, + { + name: 'jolokia.server.version', + type: 'keyword', + description: "The container's version (if detected).\n", + }, + { + name: 'jolokia.server.vendor', + type: 'keyword', + description: 'The vendor of the container the agent is running in.\n', + }, + { + name: 'jolokia.url', + type: 'keyword', + description: 'The URL how this agent can be contacted.\n', + }, + { + name: 'jolokia.secured', + type: 'boolean', + description: 'Whether the agent was configured for authentication or not.\n', + }, + ], + }, + { + key: 'log', + title: 'Log file content', + description: 'Contains log file lines.\n', + fields: [ + { + name: 'log.file.path', + type: 'keyword', + required: false, + description: + 'The file from which the line was read. This field contains the absolute path to the file. For example: `/var/log/system.log`.\n', + }, + { + name: 'log.source.address', + type: 'keyword', + required: false, + description: 'Source address from which the log event was read / sent from.\n', + }, + { + name: 'log.offset', + type: 'long', + required: false, + description: 'The file offset the reported line starts at.\n', + }, + { + name: 'stream', + type: 'keyword', + required: false, + description: "Log stream when reading container logs, can be 'stdout' or 'stderr'\n", + }, + { + name: 'input.type', + required: true, + description: + 'The input type from which the event was generated. This field is set to the value specified for the `type` option in the input section of the Filebeat config file.\n', + }, + { + name: 'syslog.facility', + type: 'long', + required: false, + description: 'The facility extracted from the priority.\n', + }, + { + name: 'syslog.priority', + type: 'long', + required: false, + description: 'The priority of the syslog event.\n', + }, + { + name: 'syslog.severity_label', + type: 'keyword', + required: false, + description: 'The human readable severity.\n', + }, + { + name: 'syslog.facility_label', + type: 'keyword', + required: false, + description: 'The human readable facility.\n', + }, + { + name: 'process.program', + type: 'keyword', + required: false, + description: 'The name of the program.\n', + }, + { + name: 'log.flags', + description: 'This field contains the flags of the event.\n', + }, + { + name: 'http.response.content_length', + type: 'alias', + path: 'http.response.body.bytes', + migration: true, + }, + { + name: 'user_agent', + type: 'group', + fields: [ + { + name: 'os', + type: 'group', + fields: [ { - name: 'flow_id', + name: 'full_name', type: 'keyword', }, - { - name: 'email', - type: 'group', - fields: [ - { - name: 'status', - type: 'keyword', - }, - ], - }, - { - name: 'dest_ip', - type: 'alias', - path: 'destination.ip', + ], + }, + ], + }, + { + name: 'fileset.name', + type: 'keyword', + description: 'The Filebeat fileset that generated this event.\n', + }, + { + name: 'fileset.module', + type: 'alias', + path: 'event.module', + migration: true, + }, + { + name: 'read_timestamp', + type: 'alias', + path: 'event.created', + migration: true, + }, + { + name: 'docker.attrs', + type: 'object', + object_type: 'keyword', + description: + "docker.attrs contains labels and environment variables written by docker's JSON File logging driver. These fields are only available when they are configured in the logging driver options.\n", + }, + { + name: 'icmp.code', + type: 'keyword', + description: 'ICMP code.\n', + }, + { + name: 'icmp.type', + type: 'keyword', + description: 'ICMP type.\n', + }, + { + name: 'igmp.type', + type: 'keyword', + description: 'IGMP type.\n', + }, + { + name: 'azure', + type: 'group', + fields: [ + { + name: 'eventhub', + type: 'keyword', + description: 'Name of the eventhub.\n', + }, + { + name: 'offset', + type: 'long', + description: 'The offset.\n', + }, + { + name: 'enqueued_time', + type: 'date', + description: 'The enqueued time.\n', + }, + { + name: 'partition_id', + type: 'long', + description: 'The partition id.\n', + }, + { + name: 'consumer_group', + type: 'keyword', + description: 'The consumer group.\n', + }, + { + name: 'sequence_number', + type: 'long', + description: 'The sequence number.\n', + }, + ], + }, + { + name: 'kafka', + type: 'group', + fields: [ + { + name: 'topic', + type: 'keyword', + description: 'Kafka topic\n', + }, + { + name: 'partition', + type: 'long', + description: 'Kafka partition number\n', + }, + { + name: 'offset', + type: 'long', + description: 'Kafka offset of this message\n', + }, + { + name: 'key', + type: 'keyword', + description: 'Kafka key, corresponding to the Kafka value stored in the message\n', + }, + { + name: 'block_timestamp', + type: 'date', + description: 'Kafka outer (compressed) block timestamp\n', + }, + { + name: 'headers', + type: 'array', + description: + 'An array of Kafka header strings for this message, in the form ": ".\n', + }, + ], + }, + ], + }, + { + key: 'apache', + title: 'Apache', + description: 'Apache Module\n', + short_config: true, + fields: [ + { + name: 'apache2', + type: 'group', + description: 'Aliases for backward compatibility with old apache2 fields\n', + fields: [ + { + name: 'access', + type: 'group', + fields: [ + { + name: 'remote_ip', + type: 'alias', + path: 'source.address', + migration: true, }, { - name: 'icmp_code', - type: 'long', + name: 'ssl.protocol', + type: 'alias', + path: 'apache.access.ssl.protocol', + migration: true, }, { - name: 'http', + name: 'ssl.cipher', + type: 'alias', + path: 'apache.access.ssl.cipher', + migration: true, + }, + { + name: 'body_sent.bytes', + type: 'alias', + path: 'http.response.body.bytes', + migration: true, + }, + { + name: 'user_name', + type: 'alias', + path: 'user.name', + migration: true, + }, + { + name: 'method', + type: 'alias', + path: 'http.request.method', + migration: true, + }, + { + name: 'url', + type: 'alias', + path: 'url.original', + migration: true, + }, + { + name: 'http_version', + type: 'alias', + path: 'http.version', + migration: true, + }, + { + name: 'response_code', + type: 'alias', + path: 'http.response.status_code', + migration: true, + }, + { + name: 'referrer', + type: 'alias', + path: 'http.request.referrer', + migration: true, + }, + { + name: 'agent', + type: 'alias', + path: 'user_agent.original', + migration: true, + }, + { + name: 'user_agent', type: 'group', fields: [ { - name: 'status', + name: 'device', type: 'alias', - path: 'http.response.status_code', + path: 'user_agent.device.name', + migration: true, }, { - name: 'redirect', - type: 'keyword', + name: 'name', + type: 'alias', + path: 'user_agent.name', + migration: true, }, { - name: 'http_user_agent', + name: 'os', type: 'alias', - path: 'user_agent.original', + path: 'user_agent.os.full_name', + migration: true, }, { - name: 'protocol', - type: 'keyword', + name: 'os_name', + type: 'alias', + path: 'user_agent.os.name', + migration: true, }, { - name: 'http_refer', + name: 'original', type: 'alias', - path: 'http.request.referrer', + path: 'user_agent.original', + migration: true, + }, + ], + }, + { + name: 'geoip', + type: 'group', + fields: [ + { + name: 'continent_name', + type: 'alias', + path: 'source.geo.continent_name', + migration: true, }, { - name: 'url', + name: 'country_iso_code', type: 'alias', - path: 'url.original', + path: 'source.geo.country_iso_code', + migration: true, }, { - name: 'hostname', + name: 'location', type: 'alias', - path: 'url.domain', + path: 'source.geo.location', + migration: true, }, { - name: 'length', + name: 'region_name', type: 'alias', - path: 'http.response.body.bytes', + path: 'source.geo.region_name', + migration: true, }, { - name: 'http_method', + name: 'city_name', type: 'alias', - path: 'http.request.method', + path: 'source.geo.city_name', + migration: true, }, { - name: 'http_content_type', - type: 'keyword', + name: 'region_iso_code', + type: 'alias', + path: 'source.geo.region_iso_code', + migration: true, }, ], }, + ], + }, + { + name: 'error', + type: 'group', + fields: [ { - name: 'timestamp', + name: 'level', type: 'alias', - path: '@timestamp', + path: 'log.level', + migration: true, }, { - name: 'in_iface', - type: 'keyword', + name: 'message', + type: 'alias', + path: 'message', + migration: true, }, { - name: 'alert', - type: 'group', - fields: [ - { - name: 'category', - type: 'keyword', - }, - { - name: 'severity', - type: 'alias', - path: 'event.severity', - }, - { - name: 'rev', - type: 'long', - }, - { - name: 'gid', - type: 'long', - }, - { - name: 'signature', - type: 'keyword', - }, - { - name: 'action', - type: 'alias', - path: 'event.outcome', - }, - { - name: 'signature_id', - type: 'long', - }, - ], + name: 'pid', + type: 'alias', + path: 'process.pid', + migration: true, }, { - name: 'ssh', - type: 'group', - fields: [ - { - name: 'client', - type: 'group', - fields: [ - { - name: 'proto_version', - type: 'keyword', - }, - { - name: 'software_version', - type: 'keyword', - }, - ], - }, - { - name: 'server', - type: 'group', - fields: [ - { - name: 'proto_version', - type: 'keyword', - }, - { - name: 'software_version', - type: 'keyword', - }, - ], - }, - ], + name: 'tid', + type: 'alias', + path: 'process.thread.id', + migration: true, }, { - name: 'stats', - type: 'group', - fields: [ - { - name: 'capture', - type: 'group', - fields: [ - { - name: 'kernel_packets', - type: 'long', - }, - { - name: 'kernel_drops', - type: 'long', - }, - { - name: 'kernel_ifdrops', - type: 'long', - }, - ], - }, - { - name: 'uptime', - type: 'long', - }, - { - name: 'detect', - type: 'group', - fields: [ - { - name: 'alert', - type: 'long', - }, - ], - }, - { - name: 'http', - type: 'group', - fields: [ - { - name: 'memcap', - type: 'long', - }, - { - name: 'memuse', - type: 'long', - }, - ], - }, - { - name: 'file_store', - type: 'group', - fields: [ - { - name: 'open_files', - type: 'long', - }, - ], - }, - { - name: 'defrag', - type: 'group', - fields: [ - { - name: 'max_frag_hits', - type: 'long', - }, - { - name: 'ipv4', - type: 'group', - fields: [ - { - name: 'timeouts', - type: 'long', - }, - { - name: 'fragments', - type: 'long', - }, - { - name: 'reassembled', - type: 'long', - }, - ], - }, - { - name: 'ipv6', - type: 'group', - fields: [ - { - name: 'timeouts', - type: 'long', - }, - { - name: 'fragments', - type: 'long', - }, - { - name: 'reassembled', - type: 'long', - }, - ], - }, - ], - }, - { - name: 'flow', - type: 'group', - fields: [ - { - name: 'tcp_reuse', - type: 'long', - }, - { - name: 'udp', - type: 'long', - }, - { - name: 'memcap', - type: 'long', - }, - { - name: 'emerg_mode_entered', - type: 'long', - }, - { - name: 'emerg_mode_over', - type: 'long', - }, - { - name: 'tcp', - type: 'long', - }, - { - name: 'icmpv6', - type: 'long', - }, - { - name: 'icmpv4', - type: 'long', - }, - { - name: 'spare', - type: 'long', - }, - { - name: 'memuse', - type: 'long', - }, - ], - }, - { - name: 'tcp', - type: 'group', - fields: [ - { - name: 'pseudo_failed', - type: 'long', - }, - { - name: 'ssn_memcap_drop', - type: 'long', - }, - { - name: 'insert_data_overlap_fail', - type: 'long', - }, - { - name: 'sessions', - type: 'long', - }, - { - name: 'pseudo', - type: 'long', - }, - { - name: 'synack', - type: 'long', - }, - { - name: 'insert_data_normal_fail', - type: 'long', - }, - { - name: 'syn', - type: 'long', - }, - { - name: 'memuse', - type: 'long', - }, - { - name: 'invalid_checksum', + name: 'module', + type: 'alias', + path: 'apache.error.module', + migration: true, + }, + ], + }, + ], + }, + { + name: 'apache', + type: 'group', + description: 'Apache fields.\n', + fields: [ + { + name: 'access', + type: 'group', + description: 'Contains fields for the Apache HTTP Server access logs.\n', + fields: [ + { + name: 'ssl.protocol', + type: 'keyword', + description: 'SSL protocol version.\n', + }, + { + name: 'ssl.cipher', + type: 'keyword', + description: 'SSL cipher name.\n', + }, + ], + }, + { + name: 'error', + type: 'group', + description: 'Fields from the Apache error logs.\n', + fields: [ + { + name: 'module', + type: 'keyword', + description: 'The module producing the logged message.\n', + }, + ], + }, + ], + }, + ], + }, + { + key: 'auditd', + title: 'Auditd', + description: 'Module for parsing auditd logs.\n', + short_config: true, + fields: [ + { + name: 'user', + type: 'group', + fields: [ + { + name: 'terminal', + type: 'keyword', + description: + 'Terminal or tty device on which the user is performing the observed activity.\n', + }, + { + name: 'audit', + type: 'group', + fields: [ + { + name: 'id', + type: 'keyword', + description: 'One or multiple unique identifiers of the user.\n', + }, + { + name: 'name', + type: 'keyword', + example: 'albert', + description: 'Short name or login of the user.\n', + }, + { + name: 'group.id', + type: 'keyword', + description: 'Unique identifier for the group on the system/platform.\n', + }, + { + name: 'group.name', + type: 'keyword', + description: 'Name of the group.\n', + }, + ], + }, + { + name: 'effective', + type: 'group', + fields: [ + { + name: 'id', + type: 'keyword', + description: 'One or multiple unique identifiers of the user.\n', + }, + { + name: 'name', + type: 'keyword', + example: 'albert', + description: 'Short name or login of the user.\n', + }, + { + name: 'group.id', + type: 'keyword', + description: 'Unique identifier for the group on the system/platform.\n', + }, + { + name: 'group.name', + type: 'keyword', + description: 'Name of the group.\n', + }, + ], + }, + { + name: 'filesystem', + type: 'group', + fields: [ + { + name: 'id', + type: 'keyword', + description: 'One or multiple unique identifiers of the user.\n', + }, + { + name: 'name', + type: 'keyword', + example: 'albert', + description: 'Short name or login of the user.\n', + }, + { + name: 'group.id', + type: 'keyword', + description: 'Unique identifier for the group on the system/platform.\n', + }, + { + name: 'group.name', + type: 'keyword', + description: 'Name of the group.\n', + }, + ], + }, + { + name: 'owner', + type: 'group', + fields: [ + { + name: 'id', + type: 'keyword', + description: 'One or multiple unique identifiers of the user.\n', + }, + { + name: 'name', + type: 'keyword', + example: 'albert', + description: 'Short name or login of the user.\n', + }, + { + name: 'group.id', + type: 'keyword', + description: 'Unique identifier for the group on the system/platform.\n', + }, + { + name: 'group.name', + type: 'keyword', + description: 'Name of the group.\n', + }, + ], + }, + { + name: 'saved', + type: 'group', + fields: [ + { + name: 'id', + type: 'keyword', + description: 'One or multiple unique identifiers of the user.\n', + }, + { + name: 'name', + type: 'keyword', + example: 'albert', + description: 'Short name or login of the user.\n', + }, + { + name: 'group.id', + type: 'keyword', + description: 'Unique identifier for the group on the system/platform.\n', + }, + { + name: 'group.name', + type: 'keyword', + description: 'Name of the group.\n', + }, + ], + }, + ], + }, + { + name: 'auditd', + type: 'group', + description: 'Fields from the auditd logs.\n', + fields: [ + { + name: 'log', + type: 'group', + description: + 'Fields from the Linux audit log. Not all fields are documented here because they are dynamic and vary by audit event type.\n', + fields: [ + { + name: 'old_auid', + description: + 'For login events this is the old audit ID used for the user prior to this login.\n', + }, + { + name: 'new_auid', + description: + 'For login events this is the new audit ID. The audit ID can be used to trace future events to the user even if their identity changes (like becoming root).\n', + }, + { + name: 'old_ses', + description: + 'For login events this is the old session ID used for the user prior to this login.\n', + }, + { + name: 'new_ses', + description: + 'For login events this is the new session ID. It can be used to tie a user to future events by session ID.\n', + }, + { + name: 'sequence', + type: 'long', + description: 'The audit event sequence number.\n', + }, + { + name: 'items', + description: 'The number of items in an event.\n', + }, + { + name: 'item', + description: + 'The item field indicates which item out of the total number of items. This number is zero-based; a value of 0 means it is the first item.\n', + }, + { + name: 'tty', + type: 'keyword', + definition: 'TTY udevice the user is running programs on.\n', + }, + { + name: 'a0', + description: 'The first argument to the system call.\n', + }, + { + name: 'addr', + type: 'ip', + definition: 'Remote address that the user is connecting from.\n', + }, + { + name: 'rport', + type: 'long', + definition: 'Remote port number.\n', + }, + { + name: 'laddr', + type: 'ip', + definition: 'Local network address.\n', + }, + { + name: 'lport', + type: 'long', + definition: 'Local port number.\n', + }, + { + name: 'acct', + type: 'alias', + path: 'user.name', + migration: true, + }, + { + name: 'pid', + type: 'alias', + path: 'process.pid', + migration: true, + }, + { + name: 'ppid', + type: 'alias', + path: 'process.ppid', + migration: true, + }, + { + name: 'res', + type: 'alias', + path: 'event.outcome', + migration: true, + }, + { + name: 'record_type', + type: 'alias', + path: 'event.action', + migration: true, + }, + { + name: 'geoip', + type: 'group', + fields: [ + { + name: 'continent_name', + type: 'alias', + path: 'source.geo.continent_name', + migration: true, + }, + { + name: 'country_iso_code', + type: 'alias', + path: 'source.geo.country_iso_code', + migration: true, + }, + { + name: 'location', + type: 'alias', + path: 'source.geo.location', + migration: true, + }, + { + name: 'region_name', + type: 'alias', + path: 'source.geo.region_name', + migration: true, + }, + { + name: 'city_name', + type: 'alias', + path: 'source.geo.city_name', + migration: true, + }, + { + name: 'region_iso_code', + type: 'alias', + path: 'source.geo.region_iso_code', + migration: true, + }, + ], + }, + { + name: 'arch', + type: 'alias', + path: 'host.architecture', + migration: true, + }, + { + name: 'gid', + type: 'alias', + path: 'user.group.id', + migration: true, + }, + { + name: 'uid', + type: 'alias', + path: 'user.id', + migration: true, + }, + { + name: 'agid', + type: 'alias', + path: 'user.audit.group.id', + migration: true, + }, + { + name: 'auid', + type: 'alias', + path: 'user.audit.id', + migration: true, + }, + { + name: 'fsgid', + type: 'alias', + path: 'user.filesystem.group.id', + migration: true, + }, + { + name: 'fsuid', + type: 'alias', + path: 'user.filesystem.id', + migration: true, + }, + { + name: 'egid', + type: 'alias', + path: 'user.effective.group.id', + migration: true, + }, + { + name: 'euid', + type: 'alias', + path: 'user.effective.id', + migration: true, + }, + { + name: 'sgid', + type: 'alias', + path: 'user.saved.group.id', + migration: true, + }, + { + name: 'suid', + type: 'alias', + path: 'user.saved.id', + migration: true, + }, + { + name: 'ogid', + type: 'alias', + path: 'user.owner.group.id', + migration: true, + }, + { + name: 'ouid', + type: 'alias', + path: 'user.owner.id', + migration: true, + }, + { + name: 'comm', + type: 'alias', + path: 'process.name', + migration: true, + }, + { + name: 'exe', + type: 'alias', + path: 'process.executable', + migration: true, + }, + { + name: 'terminal', + type: 'alias', + path: 'user.terminal', + migration: true, + }, + { + name: 'msg', + type: 'alias', + path: 'message', + migration: true, + }, + { + name: 'src', + type: 'alias', + path: 'source.address', + migration: true, + }, + { + name: 'dst', + type: 'alias', + path: 'destination.address', + migration: true, + }, + ], + }, + ], + }, + ], + }, + { + key: 'elasticsearch', + title: 'elasticsearch', + description: 'elasticsearch Module\n', + fields: [ + { + name: 'elasticsearch', + type: 'group', + description: '\n', + fields: [ + { + name: 'component', + description: 'Elasticsearch component from where the log event originated', + example: 'o.e.c.m.MetaDataCreateIndexService', + type: 'keyword', + }, + { + name: 'cluster.uuid', + description: 'UUID of the cluster', + example: 'GmvrbHlNTiSVYiPf8kxg9g', + type: 'keyword', + }, + { + name: 'cluster.name', + description: 'Name of the cluster', + example: 'docker-cluster', + type: 'keyword', + }, + { + name: 'node.id', + description: 'ID of the node', + example: 'DSiWcTyeThWtUXLB9J0BMw', + type: 'keyword', + }, + { + name: 'node.name', + description: 'Name of the node', + example: 'vWNJsZ3', + type: 'keyword', + }, + { + name: 'index.name', + description: 'Index name', + example: 'filebeat-test-input', + type: 'keyword', + }, + { + name: 'index.id', + description: 'Index id', + example: 'aOGgDwbURfCV57AScqbCgw', + type: 'keyword', + }, + { + name: 'shard.id', + description: 'Id of the shard', + example: '0', + type: 'keyword', + }, + { + name: 'audit', + type: 'group', + description: '\n', + fields: [ + { + name: 'layer', + description: + 'The layer from which this event originated: rest, transport or ip_filter', + example: 'rest', + type: 'keyword', + }, + { + name: 'event_type', + description: + 'The type of event that occurred: anonymous_access_denied, authentication_failed, access_denied, access_granted, connection_granted, connection_denied, tampered_request, run_as_granted, run_as_denied', + example: 'access_granted', + type: 'keyword', + }, + { + name: 'origin.type', + description: + 'Where the request originated: rest (request originated from a REST API request), transport (request was received on the transport channel), local_node (the local node issued the request)', + example: 'local_node', + type: 'keyword', + }, + { + name: 'realm', + description: 'The authentication realm the authentication was validated against', + example: 'default_file', + type: 'keyword', + }, + { + name: 'user.realm', + description: "The user's authentication realm, if authenticated", + example: 'active_directory', + type: 'keyword', + }, + { + name: 'user.roles', + description: 'Roles to which the principal belongs', + example: ['kibana_user', 'beats_admin'], + type: 'keyword', + }, + { + name: 'action', + description: 'The name of the action that was executed', + example: 'cluster:monitor/main', + type: 'keyword', + }, + { + name: 'url.params', + description: 'REST URI parameters', + example: '{username=jacknich2}', + }, + { + name: 'indices', + description: 'Indices accessed by action', + example: ['foo-2019.01.04', 'foo-2019.01.03', 'foo-2019.01.06'], + type: 'keyword', + }, + { + name: 'request.id', + description: 'Unique ID of request', + example: 'WzL_kb6VSvOhAq0twPvHOQ', + type: 'keyword', + }, + { + name: 'request.name', + description: 'The type of request that was executed', + example: 'ClearScrollRequest', + type: 'keyword', + }, + { + name: 'request_body', + type: 'alias', + path: 'http.request.body.content', + migration: true, + }, + { + name: 'origin_address', + type: 'alias', + path: 'source.ip', + migration: true, + }, + { + name: 'uri', + type: 'alias', + path: 'url.original', + migration: true, + }, + { + name: 'principal', + type: 'alias', + path: 'user.name', + migration: true, + }, + { + name: 'message', + type: 'text', + }, + ], + }, + { + name: 'gc', + type: 'group', + description: 'GC fileset fields.\n', + fields: [ + { + name: 'phase', + type: 'group', + description: 'Fields specific to GC phase.\n', + fields: [ + { + name: 'name', + type: 'keyword', + description: 'Name of the GC collection phase.\n', + }, + { + name: 'duration_sec', + type: 'float', + description: + 'Collection phase duration according to the Java virtual machine.\n', + }, + { + name: 'scrub_symbol_table_time_sec', + type: 'float', + description: 'Pause time in seconds cleaning up symbol tables.\n', + }, + { + name: 'scrub_string_table_time_sec', + type: 'float', + description: 'Pause time in seconds cleaning up string tables.\n', + }, + { + name: 'weak_refs_processing_time_sec', + type: 'float', + description: 'Time spent processing weak references in seconds.\n', + }, + { + name: 'parallel_rescan_time_sec', + type: 'float', + description: + 'Time spent in seconds marking live objects while application is stopped.\n', + }, + { + name: 'class_unload_time_sec', + type: 'float', + description: 'Time spent unloading unused classes in seconds.\n', + }, + { + name: 'cpu_time', + type: 'group', + description: 'Process CPU time spent performing collections.\n', + fields: [ + { + name: 'user_sec', + type: 'float', + description: 'CPU time spent outside the kernel.\n', + }, + { + name: 'sys_sec', + type: 'float', + description: 'CPU time spent inside the kernel.\n', + }, + { + name: 'real_sec', + type: 'float', + description: + 'Total elapsed CPU time spent to complete the collection from start to finish.\n', + }, + ], + }, + ], + }, + { + name: 'jvm_runtime_sec', + type: 'float', + description: 'The time from JVM start up in seconds, as a floating point number.\n', + }, + { + name: 'threads_total_stop_time_sec', + type: 'float', + description: 'Garbage collection threads total stop time seconds.\n', + }, + { + name: 'stopping_threads_time_sec', + type: 'float', + description: 'Time took to stop threads seconds.\n', + }, + { + name: 'tags', + type: 'keyword', + description: 'GC logging tags.\n', + }, + { + name: 'heap', + type: 'group', + description: 'Heap allocation and total size.\n', + fields: [ + { + name: 'size_kb', + type: 'integer', + description: 'Total heap size in kilobytes.\n', + }, + { + name: 'used_kb', + type: 'integer', + description: 'Used heap in kilobytes.\n', + }, + ], + }, + { + name: 'old_gen', + type: 'group', + description: 'Old generation occupancy and total size.\n', + fields: [ + { + name: 'size_kb', + type: 'integer', + description: 'Total size of old generation in kilobytes.\n', + }, + { + name: 'used_kb', + type: 'integer', + description: 'Old generation occupancy in kilobytes.\n', + }, + ], + }, + { + name: 'young_gen', + type: 'group', + description: 'Young generation occupancy and total size.\n', + fields: [ + { + name: 'size_kb', + type: 'integer', + description: 'Total size of young generation in kilobytes.\n', + }, + { + name: 'used_kb', + type: 'integer', + description: 'Young generation occupancy in kilobytes.\n', + }, + ], + }, + ], + }, + { + name: 'server', + description: 'Server log file', + type: 'group', + fields: [ + { + name: 'stacktrace', + description: 'Stack trace in case of errors', + index: false, + }, + { + name: 'gc', + description: 'GC log', + type: 'group', + fields: [ + { + name: 'young', + description: 'Young GC', + example: '', + type: 'group', + fields: [ + { + name: 'one', + description: '', + example: '', + type: 'long', + }, + { + name: 'two', + description: '', + example: '', + type: 'long', + }, + ], + }, + { + name: 'overhead_seq', + description: 'Sequence number', + example: 3449992, + type: 'long', + }, + { + name: 'collection_duration.ms', + description: 'Time spent in GC, in milliseconds', + example: 1600, + type: 'float', + }, + { + name: 'observation_duration.ms', + description: 'Total time over which collection was observed, in milliseconds', + example: 1800, + type: 'float', + }, + ], + }, + ], + }, + { + name: 'slowlog', + description: 'Slowlog events from Elasticsearch', + example: + '[2018-06-29T10:06:14,933][INFO ][index.search.slowlog.query] [v_VJhjV] [metricbeat-6.3.0-2018.06.26][0] took[4.5ms], took_millis[4], total_hits[19435], types[], stats[], search_type[QUERY_THEN_FETCH], total_shards[1], source[{"query":{"match_all":{"boost":1.0}}}],', + type: 'group', + fields: [ + { + name: 'logger', + description: 'Logger name', + example: 'index.search.slowlog.fetch', + type: 'keyword', + }, + { + name: 'took', + description: 'Time it took to execute the query', + example: '300ms', + type: 'keyword', + }, + { + name: 'types', + description: 'Types', + example: '', + type: 'keyword', + }, + { + name: 'stats', + description: 'Stats groups', + example: 'group1', + type: 'keyword', + }, + { + name: 'search_type', + description: 'Search type', + example: 'QUERY_THEN_FETCH', + type: 'keyword', + }, + { + name: 'source_query', + description: 'Slow query', + example: '{"query":{"match_all":{"boost":1.0}}}', + type: 'keyword', + }, + { + name: 'extra_source', + description: 'Extra source information', + example: '', + type: 'keyword', + }, + { + name: 'total_hits', + description: 'Total hits', + example: 42, + type: 'keyword', + }, + { + name: 'total_shards', + description: 'Total queried shards', + example: 22, + type: 'keyword', + }, + { + name: 'routing', + description: 'Routing', + example: 's01HZ2QBk9jw4gtgaFtn', + type: 'keyword', + }, + { + name: 'id', + description: 'Id', + example: '', + type: 'keyword', + }, + { + name: 'type', + description: 'Type', + example: 'doc', + type: 'keyword', + }, + { + name: 'source', + description: 'Source of document that was indexed', + type: 'keyword', + }, + ], + }, + ], + }, + ], + }, + { + key: 'haproxy', + title: 'haproxy', + description: 'haproxy Module\n', + fields: [ + { + name: 'haproxy', + type: 'group', + description: '\n', + fields: [ + { + name: 'frontend_name', + description: + 'Name of the frontend (or listener) which received and processed the connection.', + }, + { + name: 'backend_name', + description: + 'Name of the backend (or listener) which was selected to manage the connection to the server.', + }, + { + name: 'server_name', + description: 'Name of the last server to which the connection was sent.', + }, + { + name: 'total_waiting_time_ms', + description: 'Total time in milliseconds spent waiting in the various queues', + type: 'long', + }, + { + name: 'connection_wait_time_ms', + description: + 'Total time in milliseconds spent waiting for the connection to establish to the final server', + type: 'long', + }, + { + name: 'bytes_read', + description: 'Total number of bytes transmitted to the client when the log is emitted.', + type: 'long', + }, + { + name: 'time_queue', + description: 'Total time in milliseconds spent waiting in the various queues.', + type: 'long', + }, + { + name: 'time_backend_connect', + description: + 'Total time in milliseconds spent waiting for the connection to establish to the final server, including retries.', + type: 'long', + }, + { + name: 'server_queue', + description: + 'Total number of requests which were processed before this one in the server queue.', + type: 'long', + }, + { + name: 'backend_queue', + description: + "Total number of requests which were processed before this one in the backend's global queue.", + type: 'long', + }, + { + name: 'bind_name', + description: 'Name of the listening address which received the connection.', + }, + { + name: 'error_message', + description: 'Error message logged by HAProxy in case of error.', + type: 'text', + }, + { + name: 'source', + type: 'keyword', + description: 'The HAProxy source of the log', + }, + { + name: 'termination_state', + description: 'Condition the session was in when the session ended.', + }, + { + name: 'mode', + type: 'keyword', + description: 'mode that the frontend is operating (TCP or HTTP)', + }, + { + name: 'connections', + description: 'Contains various counts of connections active in the process.', + type: 'group', + fields: [ + { + name: 'active', + description: + 'Total number of concurrent connections on the process when the session was logged.', + type: 'long', + }, + { + name: 'frontend', + description: + 'Total number of concurrent connections on the frontend when the session was logged.', + type: 'long', + }, + { + name: 'backend', + description: + 'Total number of concurrent connections handled by the backend when the session was logged.', + type: 'long', + }, + { + name: 'server', + description: + 'Total number of concurrent connections still active on the server when the session was logged.', + type: 'long', + }, + { + name: 'retries', + description: + 'Number of connection retries experienced by this session when trying to connect to the server.', + type: 'long', + }, + ], + }, + { + name: 'client', + description: 'Information about the client doing the request', + type: 'group', + fields: [ + { + name: 'ip', + type: 'alias', + path: 'source.address', + migration: true, + }, + { + name: 'port', + type: 'alias', + path: 'source.port', + migration: true, + }, + ], + }, + { + name: 'process_name', + type: 'alias', + path: 'process.name', + migration: true, + }, + { + name: 'pid', + type: 'alias', + path: 'process.pid', + migration: true, + }, + { + name: 'destination', + description: 'Destination information', + type: 'group', + fields: [ + { + name: 'port', + type: 'alias', + path: 'destination.port', + migration: true, + }, + { + name: 'ip', + type: 'alias', + path: 'destination.ip', + migration: true, + }, + ], + }, + { + name: 'geoip', + type: 'group', + description: + 'Contains GeoIP information gathered based on the client.ip field. Only present if the GeoIP Elasticsearch plugin is available and used.\n', + fields: [ + { + name: 'continent_name', + type: 'alias', + path: 'source.geo.continent_name', + migration: true, + }, + { + name: 'country_iso_code', + type: 'alias', + path: 'source.geo.country_iso_code', + migration: true, + }, + { + name: 'location', + type: 'alias', + path: 'source.geo.location', + migration: true, + }, + { + name: 'region_name', + type: 'alias', + path: 'source.geo.region_name', + migration: true, + }, + { + name: 'city_name', + type: 'alias', + path: 'source.geo.city_name', + migration: true, + }, + { + name: 'region_iso_code', + type: 'alias', + path: 'source.geo.region_iso_code', + migration: true, + }, + ], + }, + { + name: 'http', + description: 'Please add description', + type: 'group', + fields: [ + { + name: 'response', + description: 'Fields related to the HTTP response', + type: 'group', + fields: [ + { + name: 'captured_cookie', + description: + 'Optional "name=value" entry indicating that the client had this cookie in the response.\n', + }, + { + name: 'captured_headers', + description: + 'List of headers captured in the response due to the presence of the "capture response header" statement in the frontend.\n', + type: 'keyword', + }, + { + name: 'status_code', + type: 'alias', + path: 'http.response.status_code', + migration: true, + }, + ], + }, + { + name: 'request', + description: 'Fields related to the HTTP request', + type: 'group', + fields: [ + { + name: 'captured_cookie', + description: + 'Optional "name=value" entry indicating that the server has returned a cookie with its request.\n', + }, + { + name: 'captured_headers', + description: + 'List of headers captured in the request due to the presence of the "capture request header" statement in the frontend.\n', + type: 'keyword', + }, + { + name: 'raw_request_line', + description: + 'Complete HTTP request line, including the method, request and HTTP version string.', + type: 'keyword', + }, + { + name: 'time_wait_without_data_ms', + description: + 'Total time in milliseconds spent waiting for the server to send a full HTTP response, not counting data.', + type: 'long', + }, + { + name: 'time_wait_ms', + description: + 'Total time in milliseconds spent waiting for a full HTTP request from the client (not counting body) after the first byte was received.', + type: 'long', + }, + ], + }, + ], + }, + { + name: 'tcp', + description: 'TCP log format', + type: 'group', + fields: [ + { + name: 'connection_waiting_time_ms', + type: 'long', + description: + 'Total time in milliseconds elapsed between the accept and the last close', + }, + ], + }, + ], + }, + ], + }, + { + key: 'icinga', + title: 'Icinga', + description: 'Icinga Module\n', + fields: [ + { + name: 'icinga', + type: 'group', + description: '\n', + fields: [ + { + name: 'debug', + type: 'group', + description: 'Contains fields for the Icinga debug logs.\n', + fields: [ + { + name: 'facility', + type: 'keyword', + description: 'Specifies what component of Icinga logged the message.\n', + }, + { + name: 'severity', + type: 'alias', + path: 'log.level', + migration: true, + }, + { + name: 'message', + type: 'alias', + path: 'message', + migration: true, + }, + ], + }, + { + name: 'main', + type: 'group', + description: 'Contains fields for the Icinga main logs.\n', + fields: [ + { + name: 'facility', + type: 'keyword', + description: 'Specifies what component of Icinga logged the message.\n', + }, + { + name: 'severity', + type: 'alias', + path: 'log.level', + migration: true, + }, + { + name: 'message', + type: 'alias', + path: 'message', + migration: true, + }, + ], + }, + { + name: 'startup', + type: 'group', + description: 'Contains fields for the Icinga startup logs.\n', + fields: [ + { + name: 'facility', + type: 'keyword', + description: 'Specifies what component of Icinga logged the message.\n', + }, + { + name: 'severity', + type: 'alias', + path: 'log.level', + migration: true, + }, + { + name: 'message', + type: 'alias', + path: 'message', + migration: true, + }, + ], + }, + ], + }, + ], + }, + { + key: 'iis', + title: 'IIS', + description: 'Module for parsing IIS log files.\n', + fields: [ + { + name: 'iis', + type: 'group', + description: 'Fields from IIS log files.\n', + fields: [ + { + name: 'access', + type: 'group', + description: 'Contains fields for IIS access logs.\n', + fields: [ + { + name: 'sub_status', + type: 'long', + description: 'The HTTP substatus code.\n', + }, + { + name: 'win32_status', + type: 'long', + description: 'The Windows status code.\n', + }, + { + name: 'site_name', + type: 'keyword', + description: 'The site name and instance number.\n', + }, + { + name: 'server_name', + type: 'keyword', + description: 'The name of the server on which the log file entry was generated.\n', + }, + { + name: 'cookie', + type: 'keyword', + description: 'The content of the cookie sent or received, if any.\n', + }, + { + name: 'body_received.bytes', + type: 'alias', + path: 'http.request.body.bytes', + migration: true, + }, + { + name: 'body_sent.bytes', + type: 'alias', + path: 'http.response.body.bytes', + migration: true, + }, + { + name: 'server_ip', + type: 'alias', + path: 'destination.address', + migration: true, + }, + { + name: 'method', + type: 'alias', + path: 'http.request.method', + migration: true, + }, + { + name: 'url', + type: 'alias', + path: 'url.path', + migration: true, + }, + { + name: 'query_string', + type: 'alias', + path: 'url.query', + migration: true, + }, + { + name: 'port', + type: 'alias', + path: 'destination.port', + migration: true, + }, + { + name: 'user_name', + type: 'alias', + path: 'user.name', + migration: true, + }, + { + name: 'remote_ip', + type: 'alias', + path: 'source.address', + migration: true, + }, + { + name: 'referrer', + type: 'alias', + path: 'http.request.referrer', + migration: true, + }, + { + name: 'response_code', + type: 'alias', + path: 'http.response.status_code', + migration: true, + }, + { + name: 'http_version', + type: 'alias', + path: 'http.version', + migration: true, + }, + { + name: 'hostname', + type: 'alias', + path: 'host.hostname', + migration: true, + }, + { + name: 'user_agent', + type: 'group', + fields: [ + { + name: 'device', + type: 'alias', + path: 'user_agent.device.name', + migration: true, + }, + { + name: 'name', + type: 'alias', + path: 'user_agent.name', + migration: true, + }, + { + name: 'os', + type: 'alias', + path: 'user_agent.os.full_name', + migration: true, + }, + { + name: 'os_name', + type: 'alias', + path: 'user_agent.os.name', + migration: true, + }, + { + name: 'original', + type: 'alias', + path: 'user_agent.original', + migration: true, + }, + ], + }, + { + name: 'geoip', + type: 'group', + fields: [ + { + name: 'continent_name', + type: 'alias', + path: 'source.geo.continent_name', + migration: true, + }, + { + name: 'country_iso_code', + type: 'alias', + path: 'source.geo.country_iso_code', + migration: true, + }, + { + name: 'location', + type: 'alias', + path: 'source.geo.location', + migration: true, + }, + { + name: 'region_name', + type: 'alias', + path: 'source.geo.region_name', + migration: true, + }, + { + name: 'city_name', + type: 'alias', + path: 'source.geo.city_name', + migration: true, + }, + { + name: 'region_iso_code', + type: 'alias', + path: 'source.geo.region_iso_code', + migration: true, + }, + ], + }, + ], + }, + { + name: 'error', + type: 'group', + description: 'Contains fields for IIS error logs.\n', + fields: [ + { + name: 'reason_phrase', + type: 'keyword', + description: 'The HTTP reason phrase.\n', + }, + { + name: 'queue_name', + type: 'keyword', + description: 'The IIS application pool name.\n', + }, + { + name: 'remote_ip', + type: 'alias', + path: 'source.address', + migration: true, + }, + { + name: 'remote_port', + type: 'alias', + path: 'source.port', + migration: true, + }, + { + name: 'server_ip', + type: 'alias', + path: 'destination.address', + migration: true, + }, + { + name: 'server_port', + type: 'alias', + path: 'destination.port', + migration: true, + }, + { + name: 'http_version', + type: 'alias', + path: 'http.version', + migration: true, + }, + { + name: 'method', + type: 'alias', + path: 'http.request.method', + migration: true, + }, + { + name: 'url', + type: 'alias', + path: 'url.original', + migration: true, + }, + { + name: 'response_code', + type: 'alias', + path: 'http.response.status_code', + migration: true, + }, + { + name: 'geoip', + type: 'group', + fields: [ + { + name: 'continent_name', + type: 'alias', + path: 'source.geo.continent_name', + migration: true, + }, + { + name: 'country_iso_code', + type: 'alias', + path: 'source.geo.country_iso_code', + migration: true, + }, + { + name: 'location', + type: 'alias', + path: 'source.geo.location', + migration: true, + }, + { + name: 'region_name', + type: 'alias', + path: 'source.geo.region_name', + migration: true, + }, + { + name: 'city_name', + type: 'alias', + path: 'source.geo.city_name', + migration: true, + }, + { + name: 'region_iso_code', + type: 'alias', + path: 'source.geo.region_iso_code', + migration: true, + }, + ], + }, + ], + }, + ], + }, + ], + }, + { + key: 'kafka', + title: 'Kafka', + description: 'Kafka module\n', + fields: [ + { + name: 'kafka', + type: 'group', + description: '\n', + fields: [ + { + name: 'log', + type: 'group', + description: 'Kafka log lines.\n', + fields: [ + { + name: 'level', + type: 'alias', + path: 'log.level', + migration: true, + }, + { + name: 'message', + type: 'alias', + path: 'message', + migration: true, + }, + { + name: 'component', + type: 'keyword', + description: 'Component the log is coming from.\n', + }, + { + name: 'class', + type: 'keyword', + description: 'Java class the log is coming from.\n', + }, + { + name: 'trace', + type: 'group', + description: 'Trace in the log line.\n', + fields: [ + { + name: 'class', + type: 'keyword', + description: 'Java class the trace is coming from.\n', + }, + { + name: 'message', + type: 'text', + description: 'Message part of the trace.\n', + }, + ], + }, + ], + }, + ], + }, + ], + }, + { + key: 'kibana', + title: 'kibana', + description: 'kibana Module\n', + fields: [ + { + name: 'kibana', + type: 'group', + description: '\n', + fields: [ + { + name: 'log', + type: 'group', + description: 'Kafka log lines.\n', + fields: [ + { + name: 'tags', + type: 'keyword', + description: 'Kibana logging tags.\n', + }, + { + name: 'state', + type: 'keyword', + description: 'Current state of Kibana.\n', + }, + { + name: 'meta', + type: 'object', + object_type: 'keyword', + }, + { + name: 'kibana.log.meta.req.headers.referer', + type: 'alias', + path: 'http.request.referrer', + migration: true, + }, + { + name: 'kibana.log.meta.req.referer', + type: 'alias', + path: 'http.request.referrer', + migration: true, + }, + { + name: 'kibana.log.meta.req.headers.user-agent', + type: 'alias', + path: 'user_agent.original', + migration: true, + }, + { + name: 'kibana.log.meta.req.remoteAddress', + type: 'alias', + path: 'source.address', + migration: true, + }, + { + name: 'kibana.log.meta.req.url', + type: 'alias', + path: 'url.original', + migration: true, + }, + { + name: 'kibana.log.meta.statusCode', + type: 'alias', + path: 'http.response.status_code', + migration: true, + }, + { + name: 'kibana.log.meta.method', + type: 'alias', + path: 'http.request.method', + migration: true, + }, + ], + }, + ], + }, + ], + }, + { + key: 'logstash', + title: 'logstash', + description: 'logstash Module\n', + fields: [ + { + name: 'logstash', + type: 'group', + description: '\n', + fields: [ + { + name: 'log', + title: 'Logstash', + type: 'group', + description: 'Fields from the Logstash logs.\n', + fields: [ + { + name: 'module', + type: 'keyword', + description: 'The module or class where the event originate.\n', + }, + { + name: 'thread', + type: 'keyword', + description: 'Information about the running thread where the log originate.\n', + multi_fields: [ + { + name: 'text', + type: 'text', + }, + ], + }, + { + name: 'log_event', + type: 'object', + description: 'key and value debugging information.\n', + }, + { + name: 'pipeline_id', + type: 'keyword', + example: 'main', + description: 'The ID of the pipeline.\n', + }, + { + name: 'message', + type: 'alias', + path: 'message', + migration: true, + }, + { + name: 'level', + type: 'alias', + path: 'log.level', + migration: true, + }, + ], + }, + { + name: 'slowlog', + type: 'group', + description: 'slowlog\n', + fields: [ + { + name: 'module', + type: 'keyword', + description: 'The module or class where the event originate.\n', + }, + { + name: 'thread', + type: 'keyword', + description: 'Information about the running thread where the log originate.\n', + multi_fields: [ + { + name: 'text', + type: 'text', + }, + ], + }, + { + name: 'event', + type: 'keyword', + description: 'Raw dump of the original event\n', + multi_fields: [ + { + name: 'text', + type: 'text', + }, + ], + }, + { + name: 'plugin_name', + type: 'keyword', + description: 'Name of the plugin\n', + }, + { + name: 'plugin_type', + type: 'keyword', + description: 'Type of the plugin: Inputs, Filters, Outputs or Codecs.\n', + }, + { + name: 'took_in_millis', + type: 'long', + description: 'Execution time for the plugin in milliseconds.\n', + }, + { + name: 'plugin_params', + type: 'keyword', + description: 'String value of the plugin configuration\n', + multi_fields: [ + { + name: 'text', + type: 'text', + }, + ], + }, + { + name: 'plugin_params_object', + type: 'object', + description: 'key -> value of the configuration used by the plugin.\n', + }, + { + name: 'level', + type: 'alias', + path: 'log.level', + migration: true, + }, + { + name: 'took_in_nanos', + type: 'alias', + path: 'event.duration', + migration: true, + }, + ], + }, + ], + }, + ], + }, + { + key: 'mongodb', + title: 'mongodb', + description: 'Module for parsing MongoDB log files.\n', + fields: [ + { + name: 'mongodb', + type: 'group', + description: 'Fields from MongoDB logs.\n', + fields: [ + { + name: 'log', + type: 'group', + description: 'Contains fields from MongoDB logs.\n', + fields: [ + { + name: 'component', + description: 'Functional categorization of message\n', + example: 'COMMAND', + type: 'keyword', + }, + { + name: 'context', + description: 'Context of message\n', + example: 'initandlisten', + type: 'keyword', + }, + { + name: 'severity', + type: 'alias', + path: 'log.level', + migration: true, + }, + { + name: 'message', + type: 'alias', + path: 'message', + migration: true, + }, + ], + }, + ], + }, + ], + }, + { + key: 'mysql', + title: 'MySQL', + description: 'Module for parsing the MySQL log files.\n', + short_config: true, + fields: [ + { + name: 'mysql', + type: 'group', + description: 'Fields from the MySQL log files.\n', + fields: [ + { + name: 'thread_id', + type: 'long', + description: 'The connection or thread ID for the query.\n', + }, + { + name: 'error', + type: 'group', + description: 'Contains fields from the MySQL error logs.\n', + fields: [ + { + name: 'thread_id', + type: 'alias', + path: 'mysql.thread_id', + migration: true, + }, + { + name: 'level', + type: 'alias', + path: 'log.level', + migration: true, + }, + { + name: 'message', + type: 'alias', + path: 'message', + migration: true, + }, + ], + }, + { + name: 'slowlog', + type: 'group', + description: 'Contains fields from the MySQL slow logs.\n', + fields: [ + { + name: 'lock_time.sec', + type: 'float', + description: + 'The amount of time the query waited for the lock to be available. The value is in seconds, as a floating point number.\n', + }, + { + name: 'rows_sent', + type: 'long', + description: 'The number of rows returned by the query.\n', + }, + { + name: 'rows_examined', + type: 'long', + description: 'The number of rows scanned by the query.\n', + }, + { + name: 'rows_affected', + type: 'long', + description: 'The number of rows modified by the query.\n', + }, + { + name: 'bytes_sent', + type: 'long', + format: 'bytes', + description: 'The number of bytes sent to client.\n', + }, + { + name: 'bytes_received', + type: 'long', + format: 'bytes', + description: 'The number of bytes received from client.\n', + }, + { + name: 'query', + description: 'The slow query.\n', + }, + { + name: 'id', + type: 'alias', + path: 'mysql.thread_id', + migration: true, + }, + { + name: 'schema', + type: 'keyword', + description: 'The schema where the slow query was executed.\n', + }, + { + name: 'current_user', + type: 'keyword', + description: + 'Current authenticated user, used to determine access privileges. Can differ from the value for user.\n', + }, + { + name: 'last_errno', + type: 'keyword', + description: 'Last SQL error seen.\n', + }, + { + name: 'killed', + type: 'keyword', + description: 'Code of the reason if the query was killed.\n', + }, + { + name: 'query_cache_hit', + type: 'boolean', + description: 'Whether the query cache was hit.\n', + }, + { + name: 'tmp_table', + type: 'boolean', + description: 'Whether a temporary table was used to resolve the query.\n', + }, + { + name: 'tmp_table_on_disk', + type: 'boolean', + description: 'Whether the query needed temporary tables on disk.\n', + }, + { + name: 'tmp_tables', + type: 'long', + description: 'Number of temporary tables created for this query\n', + }, + { + name: 'tmp_disk_tables', + type: 'long', + description: 'Number of temporary tables created on disk for this query.\n', + }, + { + name: 'tmp_table_sizes', + type: 'long', + format: 'bytes', + description: 'Size of temporary tables created for this query.', + }, + { + name: 'filesort', + type: 'boolean', + description: 'Whether filesort optimization was used.\n', + }, + { + name: 'filesort_on_disk', + type: 'boolean', + description: + 'Whether filesort optimization was used and it needed temporary tables on disk.\n', + }, + { + name: 'priority_queue', + type: 'boolean', + description: 'Whether a priority queue was used for filesort.\n', + }, + { + name: 'full_scan', + type: 'boolean', + description: 'Whether a full table scan was needed for the slow query.\n', + }, + { + name: 'full_join', + type: 'boolean', + description: + 'Whether a full join was needed for the slow query (no indexes were used for joins).\n', + }, + { + name: 'merge_passes', + type: 'long', + description: 'Number of merge passes executed for the query.\n', + }, + { + name: 'sort_merge_passes', + type: 'long', + description: 'Number of merge passes that the sort algorithm has had to do.\n', + }, + { + name: 'sort_range_count', + type: 'long', + description: 'Number of sorts that were done using ranges.\n', + }, + { + name: 'sort_rows', + type: 'long', + description: 'Number of sorted rows.\n', + }, + { + name: 'sort_scan_count', + type: 'long', + description: 'Number of sorts that were done by scanning the table.\n', + }, + { + name: 'log_slow_rate_type', + type: 'keyword', + description: + 'Type of slow log rate limit, it can be `session` if the rate limit is applied per session, or `query` if it applies per query.\n', + }, + { + name: 'log_slow_rate_limit', + type: 'keyword', + description: + 'Slow log rate limit, a value of 100 means that one in a hundred queries or sessions are being logged.\n', + }, + { + name: 'read_first', + type: 'long', + description: 'The number of times the first entry in an index was read.\n', + }, + { + name: 'read_last', + type: 'long', + description: 'The number of times the last key in an index was read.\n', + }, + { + name: 'read_key', + type: 'long', + description: 'The number of requests to read a row based on a key.\n', + }, + { + name: 'read_next', + type: 'long', + description: 'The number of requests to read the next row in key order.\n', + }, + { + name: 'read_prev', + type: 'long', + description: 'The number of requests to read the previous row in key order.\n', + }, + { + name: 'read_rnd', + type: 'long', + description: 'The number of requests to read a row based on a fixed position.\n', + }, + { + name: 'read_rnd_next', + type: 'long', + description: 'The number of requests to read the next row in the data file.\n', + }, + { + name: 'innodb', + type: 'group', + description: 'Contains fields relative to InnoDB engine\n', + fields: [ + { + name: 'trx_id', + type: 'keyword', + description: 'Transaction ID\n', + }, + { + name: 'io_r_ops', + type: 'long', + description: 'Number of page read operations.\n', + }, + { + name: 'io_r_bytes', + type: 'long', + format: 'bytes', + description: 'Bytes read during page read operations.\n', + }, + { + name: 'io_r_wait.sec', + type: 'long', + description: 'How long it took to read all needed data from storage.\n', + }, + { + name: 'rec_lock_wait.sec', + type: 'long', + description: 'How long the query waited for locks.\n', + }, + { + name: 'queue_wait.sec', + type: 'long', + description: + 'How long the query waited to enter the InnoDB queue and to be executed once in the queue.\n', + }, + { + name: 'pages_distinct', + type: 'long', + description: 'Approximated count of pages accessed to execute the query.\n', + }, + ], + }, + { + name: 'user', + type: 'alias', + path: 'user.name', + migration: true, + }, + { + name: 'host', + type: 'alias', + path: 'source.domain', + migration: true, + }, + { + name: 'ip', + type: 'alias', + path: 'source.ip', + migration: true, + }, + ], + }, + ], + }, + ], + }, + { + key: 'nats', + title: 'nats', + description: 'Module for parsing NATS log files.\n', + release: 'beta', + fields: [ + { + name: 'nats', + type: 'group', + description: 'Fields from NATS logs.\n', + fields: [ + { + name: 'log', + type: 'group', + description: 'Nats log files\n', + release: 'beta', + fields: [ + { + name: 'client', + type: 'group', + description: 'Fields from NATS logs client.\n', + fields: [ + { + name: 'id', + type: 'integer', + description: 'The id of the client\n', + }, + ], + }, + { + name: 'msg', + type: 'group', + description: 'Fields from NATS logs message.\n', + fields: [ + { + name: 'bytes', + type: 'long', + format: 'bytes', + description: 'Size of the payload in bytes\n', + }, + { + name: 'type', + type: 'keyword', + description: 'The protocol message type\n', + }, + { + name: 'subject', + type: 'keyword', + description: 'Subject name this message was received on\n', + }, + { + name: 'sid', + type: 'integer', + description: 'The unique alphanumeric subscription ID of the subject\n', + }, + { + name: 'reply_to', + type: 'keyword', + description: + 'The inbox subject on which the publisher is listening for responses\n', + }, + { + name: 'max_messages', + type: 'integer', + description: + 'An optional number of messages to wait for before automatically unsubscribing\n', + }, + { + name: 'error.message', + type: 'text', + description: 'Details about the error occurred\n', + }, + { + name: 'queue_group', + type: 'text', + description: 'The queue group which subscriber will join\n', + }, + ], + }, + ], + }, + ], + }, + ], + }, + { + key: 'nginx', + title: 'Nginx', + description: 'Module for parsing the Nginx log files.\n', + short_config: true, + fields: [ + { + name: 'nginx', + type: 'group', + description: 'Fields from the Nginx log files.\n', + fields: [ + { + name: 'access', + type: 'group', + description: 'Contains fields for the Nginx access logs.\n', + fields: [ + { + name: 'remote_ip_list', + type: 'array', + description: + 'An array of remote IP addresses. It is a list because it is common to include, besides the client IP address, IP addresses from headers like `X-Forwarded-For`. Real source IP is restored to `source.ip`.\n', + }, + { + name: 'body_sent.bytes', + type: 'alias', + path: 'http.response.body.bytes', + migration: true, + }, + { + name: 'user_name', + type: 'alias', + path: 'user.name', + migration: true, + }, + { + name: 'method', + type: 'alias', + path: 'http.request.method', + migration: true, + }, + { + name: 'url', + type: 'alias', + path: 'url.original', + migration: true, + }, + { + name: 'http_version', + type: 'alias', + path: 'http.version', + migration: true, + }, + { + name: 'response_code', + type: 'alias', + path: 'http.response.status_code', + migration: true, + }, + { + name: 'referrer', + type: 'alias', + path: 'http.request.referrer', + migration: true, + }, + { + name: 'agent', + type: 'alias', + path: 'user_agent.original', + migration: true, + }, + { + name: 'user_agent', + type: 'group', + fields: [ + { + name: 'device', + type: 'alias', + path: 'user_agent.device.name', + migration: true, + }, + { + name: 'name', + type: 'alias', + path: 'user_agent.name', + migration: true, + }, + { + name: 'os', + type: 'alias', + path: 'user_agent.os.full_name', + migration: true, + }, + { + name: 'os_name', + type: 'alias', + path: 'user_agent.os.name', + migration: true, + }, + { + name: 'original', + type: 'alias', + path: 'user_agent.original', + migration: true, + }, + ], + }, + { + name: 'geoip', + type: 'group', + fields: [ + { + name: 'continent_name', + type: 'alias', + path: 'source.geo.continent_name', + migration: true, + }, + { + name: 'country_iso_code', + type: 'alias', + path: 'source.geo.country_iso_code', + migration: true, + }, + { + name: 'location', + type: 'alias', + path: 'source.geo.location', + migration: true, + }, + { + name: 'region_name', + type: 'alias', + path: 'source.geo.region_name', + migration: true, + }, + { + name: 'city_name', + type: 'alias', + path: 'source.geo.city_name', + migration: true, + }, + { + name: 'region_iso_code', + type: 'alias', + path: 'source.geo.region_iso_code', + migration: true, + }, + ], + }, + ], + }, + { + name: 'error', + type: 'group', + description: 'Contains fields for the Nginx error logs.\n', + fields: [ + { + name: 'connection_id', + type: 'long', + description: 'Connection identifier.\n', + }, + { + name: 'level', + type: 'alias', + path: 'log.level', + migration: true, + }, + { + name: 'pid', + type: 'alias', + path: 'process.pid', + migration: true, + }, + { + name: 'tid', + type: 'alias', + path: 'process.thread.id', + migration: true, + }, + { + name: 'message', + type: 'alias', + path: 'message', + migration: true, + }, + ], + }, + { + name: 'ingress_controller', + type: 'group', + description: 'Contains fields for the Ingress Nginx controller access logs.\n', + fields: [ + { + name: 'remote_ip_list', + type: 'array', + description: + 'An array of remote IP addresses. It is a list because it is common to include, besides the client IP address, IP addresses from headers like `X-Forwarded-For`. Real source IP is restored to `source.ip`.\n', + }, + { + name: 'http.request.length', + type: 'long', + format: 'bytes', + description: + 'The request length (including request line, header, and request body)\n', + }, + { + name: 'http.request.time', + type: 'double', + format: 'duration', + description: 'Time elapsed since the first bytes were read from the client\n', + }, + { + name: 'upstream.name', + type: 'text', + description: 'The name of the upstream.\n', + }, + { + name: 'upstream.alternative_name', + type: 'text', + description: 'The name of the alternative upstream.\n', + }, + { + name: 'upstream.response.length', + type: 'long', + format: 'bytes', + description: 'The length of the response obtained from the upstream server\n', + }, + { + name: 'upstream.response.time', + type: 'double', + format: 'duration', + description: + 'The time spent on receiving the response from the upstream server as seconds with millisecond resolution\n', + }, + { + name: 'upstream.response.status_code', + type: 'long', + description: 'The status code of the response obtained from the upstream server\n', + }, + { + name: 'http.request.id', + type: 'text', + description: 'The randomly generated ID of the request\n', + }, + { + name: 'upstream.ip', + type: 'ip', + description: + 'The IP address of the upstream server. If several servers were contacted during request processing, their addresses are separated by commas.\n', + }, + { + name: 'upstream.port', + type: 'long', + description: 'The port of the upstream server.\n', + }, + { + name: 'body_sent.bytes', + type: 'alias', + path: 'http.response.body.bytes', + migration: true, + }, + { + name: 'user_name', + type: 'alias', + path: 'user.name', + migration: true, + }, + { + name: 'method', + type: 'alias', + path: 'http.request.method', + migration: true, + }, + { + name: 'url', + type: 'alias', + path: 'url.original', + migration: true, + }, + { + name: 'http_version', + type: 'alias', + path: 'http.version', + migration: true, + }, + { + name: 'response_code', + type: 'alias', + path: 'http.response.status_code', + migration: true, + }, + { + name: 'referrer', + type: 'alias', + path: 'http.request.referrer', + migration: true, + }, + { + name: 'agent', + type: 'alias', + path: 'user_agent.original', + migration: true, + }, + { + name: 'user_agent', + type: 'group', + fields: [ + { + name: 'device', + type: 'alias', + path: 'user_agent.device.name', + migration: true, + }, + { + name: 'name', + type: 'alias', + path: 'user_agent.name', + migration: true, + }, + { + name: 'os', + type: 'alias', + path: 'user_agent.os.full_name', + migration: true, + }, + { + name: 'os_name', + type: 'alias', + path: 'user_agent.os.name', + migration: true, + }, + { + name: 'original', + type: 'alias', + path: 'user_agent.original', + migration: true, + }, + ], + }, + { + name: 'geoip', + type: 'group', + fields: [ + { + name: 'continent_name', + type: 'alias', + path: 'source.geo.continent_name', + migration: true, + }, + { + name: 'country_iso_code', + type: 'alias', + path: 'source.geo.country_iso_code', + migration: true, + }, + { + name: 'location', + type: 'alias', + path: 'source.geo.location', + migration: true, + }, + { + name: 'region_name', + type: 'alias', + path: 'source.geo.region_name', + migration: true, + }, + { + name: 'city_name', + type: 'alias', + path: 'source.geo.city_name', + migration: true, + }, + { + name: 'region_iso_code', + type: 'alias', + path: 'source.geo.region_iso_code', + migration: true, + }, + ], + }, + ], + }, + ], + }, + ], + }, + { + key: 'osquery', + title: 'Osquery', + description: 'Fields exported by the `osquery` module\n', + fields: [ + { + name: 'osquery', + type: 'group', + description: '\n', + fields: [ + { + name: 'result', + type: 'group', + description: 'Common fields exported by the result metricset.\n', + fields: [ + { + name: 'name', + type: 'keyword', + description: 'The name of the query that generated this event.\n', + }, + { + name: 'action', + type: 'keyword', + description: + 'For incremental data, marks whether the entry was added or removed. It can be one of "added", "removed", or "snapshot".\n', + }, + { + name: 'host_identifier', + type: 'keyword', + description: + 'The identifier for the host on which the osquery agent is running. Normally the hostname.\n', + }, + { + name: 'unix_time', + type: 'long', + description: + 'Unix timestamp of the event, in seconds since the epoch. Used for computing the `@timestamp` column.\n', + }, + { + name: 'calendar_time', + type: 'keyword', + description: + 'String representation of the collection time, as formatted by osquery.\n', + }, + ], + }, + ], + }, + ], + }, + { + key: 'postgresql', + title: 'PostgreSQL', + description: 'Module for parsing the PostgreSQL log files.\n', + short_config: true, + fields: [ + { + name: 'postgresql', + type: 'group', + description: 'Fields from PostgreSQL logs.\n', + fields: [ + { + name: 'log', + type: 'group', + description: 'Fields from the PostgreSQL log files.\n', + fields: [ + { + name: 'timestamp', + deprecated: '7.3.0', + description: 'The timestamp from the log line.\n', + }, + { + name: 'core_id', + type: 'long', + description: 'Core id\n', + }, + { + name: 'database', + example: 'mydb', + description: 'Name of database\n', + }, + { + name: 'query', + example: 'SELECT * FROM users;', + description: 'Query statement.\n', + }, + { + name: 'query_step', + example: 'parse', + description: + 'Statement step when using extended query protocol (one of statement, parse, bind or execute)\n', + }, + { + name: 'query_name', + example: 'pdo_stmt_00000001', + description: + 'Name given to a query when using extended query protocol. If it is "", or not present, this field is ignored.\n', + }, + { + name: 'error.code', + type: 'long', + description: 'Error code returned by Postgres (if any)', + }, + { + name: 'timezone', + type: 'alias', + path: 'event.timezone', + migration: true, + }, + { + name: 'thread_id', + type: 'alias', + path: 'process.pid', + migration: true, + }, + { + name: 'user', + type: 'alias', + path: 'user.name', + migration: true, + }, + { + name: 'level', + type: 'alias', + path: 'log.level', + migration: true, + }, + { + name: 'message', + type: 'alias', + path: 'message', + migration: true, + }, + ], + }, + ], + }, + ], + }, + { + key: 'redis', + title: 'Redis', + description: 'Redis Module\n', + fields: [ + { + name: 'redis', + type: 'group', + description: '\n', + fields: [ + { + name: 'log', + type: 'group', + description: 'Redis log files\n', + fields: [ + { + name: 'role', + type: 'keyword', + description: + 'The role of the Redis instance. Can be one of `master`, `slave`, `child` (for RDF/AOF writing child), or `sentinel`.\n', + }, + { + name: 'pid', + type: 'alias', + path: 'process.pid', + migration: true, + }, + { + name: 'level', + type: 'alias', + path: 'log.level', + migration: true, + }, + { + name: 'message', + type: 'alias', + path: 'message', + migration: true, + }, + ], + }, + { + name: 'slowlog', + type: 'group', + description: 'Slow logs are retrieved from Redis via a network connection.\n', + fields: [ + { + name: 'cmd', + type: 'keyword', + description: 'The command executed.\n', + }, + { + name: 'duration.us', + type: 'long', + description: 'How long it took to execute the command in microseconds.\n', + }, + { + name: 'id', + type: 'long', + description: 'The ID of the query.\n', + }, + { + name: 'key', + type: 'keyword', + description: 'The key on which the command was executed.\n', + }, + { + name: 'args', + type: 'keyword', + description: 'The arguments with which the command was called.\n', + }, + ], + }, + ], + }, + ], + }, + { + key: 'santa', + title: 'Google Santa', + description: 'Santa Module\n', + fields: [ + { + name: 'santa', + type: 'group', + description: '\n', + fields: [ + { + name: 'action', + type: 'keyword', + example: 'EXEC', + description: 'Action', + }, + { + name: 'decision', + type: 'keyword', + example: 'ALLOW', + description: 'Decision that santad took.', + }, + { + name: 'reason', + type: 'keyword', + example: 'CERT', + description: 'Reason for the decsision.', + }, + { + name: 'mode', + type: 'keyword', + example: 'M', + description: 'Operating mode of Santa.', + }, + { + name: 'disk', + type: 'group', + description: 'Fields for DISKAPPEAR actions.', + fields: [ + { + name: 'volume', + description: 'The volume name.', + }, + { + name: 'bus', + description: 'The disk bus protocol.', + }, + { + name: 'serial', + description: 'The disk serial number.', + }, + { + name: 'bsdname', + example: 'disk1s3', + description: 'The disk BSD name.', + }, + { + name: 'model', + example: 'APPLE SSD SM0512L', + description: 'The disk model.', + }, + { + name: 'fs', + example: 'apfs', + description: 'The disk volume kind (filesystem type).', + }, + { + name: 'mount', + description: 'The disk volume path.', + }, + ], + }, + ], + }, + { + name: 'certificate.common_name', + type: 'keyword', + description: 'Common name from code signing certificate.', + }, + { + name: 'certificate.sha256', + type: 'keyword', + description: 'SHA256 hash of code signing certificate.', + }, + ], + }, + { + key: 'system', + title: 'System', + description: 'Module for parsing system log files.\n', + short_config: true, + fields: [ + { + name: 'system', + type: 'group', + description: 'Fields from the system log files.\n', + fields: [ + { + name: 'auth', + type: 'group', + description: 'Fields from the Linux authorization logs.\n', + fields: [ + { + name: 'timestamp', + type: 'alias', + path: '@timestamp', + migration: true, + }, + { + name: 'hostname', + type: 'alias', + path: 'host.hostname', + migration: true, + }, + { + name: 'program', + type: 'alias', + path: 'process.name', + migration: true, + }, + { + name: 'pid', + type: 'alias', + path: 'process.pid', + migration: true, + }, + { + name: 'message', + type: 'alias', + path: 'message', + migration: true, + }, + { + name: 'user', + type: 'alias', + path: 'user.name', + migration: true, + }, + { + name: 'ssh', + type: 'group', + fields: [ + { + name: 'method', + description: + 'The SSH authentication method. Can be one of "password" or "publickey".\n', + }, + { + name: 'signature', + description: 'The signature of the client public key.\n', + }, + { + name: 'dropped_ip', + type: 'ip', + description: + 'The client IP from SSH connections that are open and immediately dropped.\n', + }, + { + name: 'event', + example: 'Accepted', + description: + 'The SSH event as found in the logs (Accepted, Invalid, Failed, etc.)\n', + }, + { + name: 'ip', + type: 'alias', + path: 'source.ip', + migration: true, + }, + { + name: 'port', + type: 'alias', + path: 'source.port', + migration: true, + }, + { + name: 'geoip', + type: 'group', + fields: [ + { + name: 'continent_name', + type: 'alias', + path: 'source.geo.continent_name', + migration: true, + }, + { + name: 'country_iso_code', + type: 'alias', + path: 'source.geo.country_iso_code', + migration: true, + }, + { + name: 'location', + type: 'alias', + path: 'source.geo.location', + migration: true, + }, + { + name: 'region_name', + type: 'alias', + path: 'source.geo.region_name', + migration: true, + }, + { + name: 'city_name', + type: 'alias', + path: 'source.geo.city_name', + migration: true, + }, + { + name: 'region_iso_code', + type: 'alias', + path: 'source.geo.region_iso_code', + migration: true, + }, + ], + }, + ], + }, + { + name: 'sudo', + type: 'group', + description: 'Fields specific to events created by the `sudo` command.\n', + fields: [ + { + name: 'error', + example: 'user NOT in sudoers', + description: 'The error message in case the sudo command failed.\n', + }, + { + name: 'tty', + description: 'The TTY where the sudo command is executed.\n', + }, + { + name: 'pwd', + description: 'The current directory where the sudo command is executed.\n', + }, + { + name: 'user', + example: 'root', + description: 'The target user to which the sudo command is switching.\n', + }, + { + name: 'command', + description: 'The command executed via sudo.\n', + }, + ], + }, + { + name: 'useradd', + type: 'group', + description: 'Fields specific to events created by the `useradd` command.\n', + fields: [ + { + name: 'home', + description: 'The home folder for the new user.', + }, + { + name: 'shell', + description: 'The default shell for the new user.', + }, + { + name: 'name', + type: 'alias', + path: 'user.name', + migration: true, + }, + { + name: 'uid', + type: 'alias', + path: 'user.id', + migration: true, + }, + { + name: 'gid', + type: 'alias', + path: 'group.id', + migration: true, + }, + ], + }, + { + name: 'groupadd', + type: 'group', + description: 'Fields specific to events created by the `groupadd` command.\n', + fields: [ + { + name: 'name', + type: 'alias', + path: 'group.name', + migration: true, + }, + { + name: 'gid', + type: 'alias', + path: 'group.id', + migration: true, + }, + ], + }, + ], + }, + { + name: 'syslog', + type: 'group', + description: 'Contains fields from the syslog system logs.\n', + fields: [ + { + name: 'timestamp', + type: 'alias', + path: '@timestamp', + migration: true, + }, + { + name: 'hostname', + type: 'alias', + path: 'host.hostname', + migration: true, + }, + { + name: 'program', + type: 'alias', + path: 'process.name', + migration: true, + }, + { + name: 'pid', + type: 'alias', + path: 'process.pid', + migration: true, + }, + { + name: 'message', + type: 'alias', + path: 'message', + migration: true, + }, + ], + }, + ], + }, + ], + }, + { + key: 'traefik', + title: 'Traefik', + description: 'Module for parsing the Traefik log files.\n', + fields: [ + { + name: 'traefik', + type: 'group', + description: 'Fields from the Traefik log files.\n', + fields: [ + { + name: 'access', + type: 'group', + description: 'Contains fields for the Traefik access logs.\n', + fields: [ + { + name: 'user_identifier', + type: 'keyword', + description: 'Is the RFC 1413 identity of the client\n', + }, + { + name: 'request_count', + type: 'long', + description: 'The number of requests\n', + }, + { + name: 'frontend_name', + type: 'keyword', + description: 'The name of the frontend used\n', + }, + { + name: 'backend_url', + type: 'keyword', + description: 'The url of the backend where request is forwarded', + }, + { + name: 'body_sent.bytes', + type: 'alias', + path: 'http.response.body.bytes', + migration: true, + }, + { + name: 'remote_ip', + type: 'alias', + path: 'source.address', + migration: true, + }, + { + name: 'user_name', + type: 'alias', + path: 'user.name', + migration: true, + }, + { + name: 'method', + type: 'alias', + path: 'http.request.method', + migration: true, + }, + { + name: 'url', + type: 'alias', + path: 'url.original', + migration: true, + }, + { + name: 'http_version', + type: 'alias', + path: 'http.version', + migration: true, + }, + { + name: 'response_code', + type: 'alias', + path: 'http.response.status_code', + migration: true, + }, + { + name: 'referrer', + type: 'alias', + path: 'http.request.referrer', + migration: true, + }, + { + name: 'agent', + type: 'alias', + path: 'user_agent.original', + migration: true, + }, + { + name: 'user_agent', + type: 'group', + fields: [ + { + name: 'device', + type: 'alias', + path: 'user_agent.device.name', + }, + { + name: 'name', + type: 'alias', + path: 'user_agent.name', + }, + { + name: 'os', + type: 'alias', + path: 'user_agent.os.full_name', + }, + { + name: 'os_name', + type: 'alias', + path: 'user_agent.os.name', + }, + { + name: 'original', + type: 'alias', + path: 'user_agent.original', + }, + ], + }, + { + name: 'geoip', + type: 'group', + fields: [ + { + name: 'continent_name', + type: 'alias', + path: 'source.geo.continent_name', + }, + { + name: 'country_iso_code', + type: 'alias', + path: 'source.geo.country_iso_code', + }, + { + name: 'location', + type: 'alias', + path: 'source.geo.location', + }, + { + name: 'region_name', + type: 'alias', + path: 'source.geo.region_name', + }, + { + name: 'city_name', + type: 'alias', + path: 'source.geo.city_name', + }, + { + name: 'region_iso_code', + type: 'alias', + path: 'source.geo.region_iso_code', + }, + ], + }, + ], + }, + ], + }, + ], + }, + { + key: 'activemq', + title: 'activemq', + release: 'ga', + description: 'Module for parsing ActiveMQ log files.\n', + fields: [ + { + name: 'activemq', + type: 'group', + description: '\n', + fields: [ + { + name: 'caller', + type: 'keyword', + description: 'Name of the caller issuing the logging request (class or resource).\n', + }, + { + name: 'thread', + type: 'keyword', + description: 'Thread that generated the logging event.\n', + }, + { + name: 'user', + type: 'keyword', + description: 'User that generated the logging event.\n', + }, + { + name: 'audit', + type: 'group', + description: 'Fields from ActiveMQ audit logs.\n', + fields: [], + }, + { + name: 'log', + type: 'group', + description: 'Fields from ActiveMQ application logs.\n', + fields: [ + { + name: 'stack_trace', + type: 'keyword', + }, + ], + }, + ], + }, + ], + }, + { + key: 'aws', + title: 'AWS', + release: 'beta', + description: 'Module for handling logs from AWS.\n', + fields: [ + { + name: 'aws', + type: 'group', + description: 'Fields from AWS logs.\n', + fields: [ + { + name: 'cloudtrail', + type: 'group', + release: 'beta', + default_field: false, + description: 'Fields for AWS CloudTrail logs.\n', + fields: [ + { + name: 'event_version', + type: 'keyword', + description: 'The CloudTrail version of the log event format.\n', + }, + { + name: 'user_identity', + type: 'group', + description: + 'The userIdentity element contains details about the type of IAM identity that made the request, and which credentials were used. If temporary credentials were used, the element shows how the credentials were obtained.', + fields: [ + { + name: 'type', + type: 'keyword', + description: 'The type of the identity\n', + }, + { + name: 'arn', + type: 'keyword', + description: + 'The Amazon Resource Name (ARN) of the principal that made the call.', + }, + { + name: 'access_key_id', + type: 'keyword', + description: 'The access key ID that was used to sign the request.', + }, + { + name: 'session_context', + type: 'group', + description: + 'If the request was made with temporary security credentials, an element that provides information about the session that was created for those credentials', + fields: [ + { + name: 'mfa_authenticated', + type: 'keyword', + description: + 'The value is true if the root user or IAM user whose credentials were used for the request also was authenticated with an MFA device; otherwise, false.', + }, + { + name: 'creation_date', + type: 'date', + description: + 'The date and time when the temporary security credentials were issued.', + }, + ], + }, + { + name: 'invoked_by', + type: 'keyword', + description: + 'The name of the AWS service that made the request, such as Amazon EC2 Auto Scaling or AWS Elastic Beanstalk.', + }, + { + name: 'session_issuer', + type: 'group', + description: + 'If the request was made with temporary security credentials, an element that provides information about how the credentials were obtained.', + fields: [ + { + name: 'type', + type: 'keyword', + description: + 'The source of the temporary security credentials, such as Root, IAMUser, or Role.', + }, + { + name: 'principal_id', + type: 'keyword', + description: + 'The internal ID of the entity that was used to get credentials.', + }, + { + name: 'arn', + type: 'keyword', + description: + 'The ARN of the source (account, IAM user, or role) that was used to get temporary security credentials.', + }, + { + name: 'account_id', + type: 'keyword', + description: + 'The account that owns the entity that was used to get credentials.', + }, + ], + }, + ], + }, + { + name: 'error_code', + type: 'keyword', + description: 'The AWS service error if the request returns an error.', + }, + { + name: 'error_message', + type: 'keyword', + description: 'If the request returns an error, the description of the error.', + }, + { + name: 'request_parameters', + type: 'keyword', + description: 'The parameters, if any, that were sent with the request.', + }, + { + name: 'response_elements', + type: 'keyword', + description: + 'The response element for actions that make changes (create, update, or delete actions).', + }, + { + name: 'additional_eventdata', + type: 'keyword', + description: + 'Additional data about the event that was not part of the request or response.', + }, + { + name: 'request_id', + type: 'keyword', + description: + 'The value that identifies the request. The service being called generates this value.', + }, + { + name: 'event_type', + type: 'keyword', + description: 'Identifies the type of event that generated the event record.', + }, + { + name: 'api_version', + type: 'keyword', + description: + 'Identifies the API version associated with the AwsApiCall eventType value.', + }, + { + name: 'management_event', + type: 'keyword', + description: + 'A Boolean value that identifies whether the event is a management event.', + }, + { + name: 'read_only', + type: 'keyword', + description: 'Identifies whether this operation is a read-only operation.', + }, + { + name: 'resources', + type: 'group', + description: 'A list of resources accessed in the event.', + fields: [ + { + name: 'arn', + type: 'keyword', + description: 'Resource ARNs', + }, + { + name: 'account_id', + type: 'keyword', + description: 'Account ID of the resource owner', + }, + { + name: 'type', + type: 'keyword', + description: + 'Resource type identifier in the format: AWS::aws-service-name::data-type-name', + }, + ], + }, + { + name: 'recipient_account_id', + type: 'keyword', + description: 'Represents the account ID that received this event.', + }, + { + name: 'service_event_details', + type: 'keyword', + description: + 'Identifies the service event, including what triggered the event and the result.', + }, + { + name: 'shared_event_id', + type: 'keyword', + description: + 'GUID generated by CloudTrail to uniquely identify CloudTrail events from the same AWS action that is sent to different AWS accounts.', + }, + { + name: 'vpc_endpoint_id', + type: 'keyword', + description: + 'Identifies the VPC endpoint in which requests were made from a VPC to another AWS service, such as Amazon S3.', + }, + { + name: 'console_login', + type: 'group', + description: 'Fields specific to ConsoleLogin events', + fields: [ + { + name: 'additional_eventdata', + type: 'group', + description: 'Additional Event Data for ConsoleLogin events\n', + fields: [ + { + name: 'mobile_version', + type: 'boolean', + description: 'Identifies whether ConsoleLogin was from mobile version', + }, + { + name: 'login_to', + type: 'keyword', + description: 'URL for ConsoleLogin', + }, + { + name: 'mfa_used', + type: 'boolean', + description: + 'Identifies whether multi factor authentication was used during ConsoleLogin', + }, + ], + }, + ], + }, + ], + }, + { + name: 'cloudwatch', + type: 'group', + release: 'beta', + default_field: false, + description: 'Fields for AWS CloudWatch logs.\n', + fields: [], + }, + { + name: 'ec2', + type: 'group', + release: 'beta', + default_field: false, + description: 'Fields for AWS EC2 logs in CloudWatch.\n', + fields: [ + { + name: 'ip_address', + type: 'keyword', + description: 'The internet address of the requester.\n', + }, + ], + }, + { + name: 'elb', + type: 'group', + release: 'ga', + description: 'Fields for AWS ELB logs.\n', + fields: [ + { + name: 'name', + type: 'keyword', + description: 'The name of the load balancer.\n', + }, + { + name: 'type', + type: 'keyword', + description: 'The type of the load balancer for v2 Load Balancers.\n', + }, + { + name: 'target_group.arn', + type: 'keyword', + description: 'The ARN of the target group handling the request.\n', + }, + { + name: 'listener', + type: 'keyword', + description: 'The ELB listener that received the connection.\n', + }, + { + name: 'protocol', + type: 'keyword', + description: 'The protocol of the load balancer (http or tcp).\n', + }, + { + name: 'request_processing_time.sec', + type: 'float', + description: + 'The total time in seconds since the connection or request is received until it is sent to a registered backend.\n', + }, + { + name: 'backend_processing_time.sec', + type: 'float', + description: + 'The total time in seconds since the connection is sent to the backend till the backend starts responding.\n', + }, + { + name: 'response_processing_time.sec', + type: 'float', + description: + 'The total time in seconds since the response is received from the backend till it is sent to the client.\n', + }, + { + name: 'connection_time.ms', + type: 'long', + description: + 'The total time of the connection in milliseconds, since it is opened till it is closed.\n', + }, + { + name: 'tls_handshake_time.ms', + type: 'long', + description: + 'The total time for the TLS handshake to complete in milliseconds once the connection has been established.\n', + }, + { + name: 'backend.ip', + type: 'keyword', + description: 'The IP address of the backend processing this connection.\n', + }, + { + name: 'backend.port', + type: 'keyword', + description: 'The port in the backend processing this connection.\n', + }, + { + name: 'backend.http.response.status_code', + type: 'keyword', + description: + 'The status code from the backend (status code sent to the client from ELB is stored in `http.response.status_code`\n', + }, + { + name: 'ssl_cipher', + type: 'keyword', + description: 'The SSL cipher used in TLS/SSL connections.\n', + }, + { + name: 'ssl_protocol', + type: 'keyword', + description: 'The SSL protocol used in TLS/SSL connections.\n', + }, + { + name: 'chosen_cert.arn', + type: 'keyword', + description: + 'The ARN of the chosen certificate presented to the client in TLS/SSL connections.\n', + }, + { + name: 'chosen_cert.serial', + type: 'keyword', + description: + 'The serial number of the chosen certificate presented to the client in TLS/SSL connections.\n', + }, + { + name: 'incoming_tls_alert', + type: 'keyword', + description: + 'The integer value of TLS alerts received by the load balancer from the client, if present.\n', + }, + { + name: 'tls_named_group', + type: 'keyword', + description: 'The TLS named group.\n', + }, + { + name: 'trace_id', + type: 'keyword', + description: 'The contents of the `X-Amzn-Trace-Id` header.\n', + }, + { + name: 'matched_rule_priority', + type: 'keyword', + description: + 'The priority value of the rule that matched the request, if a rule matched.\n', + }, + { + name: 'action_executed', + type: 'keyword', + description: + 'The action executed when processing the request (forward, fixed-response, authenticate...). It can contain several values.\n', + }, + { + name: 'redirect_url', + type: 'keyword', + description: 'The URL used if a redirection action was executed.\n', + }, + { + name: 'error.reason', + type: 'keyword', + description: 'The error reason if the executed action failed.', + }, + ], + }, + { + name: 's3access', + type: 'group', + release: 'ga', + description: 'Fields for AWS S3 server access logs.\n', + fields: [ + { + name: 'bucket_owner', + type: 'keyword', + description: 'The canonical user ID of the owner of the source bucket.\n', + }, + { + name: 'bucket', + type: 'keyword', + description: 'The name of the bucket that the request was processed against.\n', + }, + { + name: 'remote_ip', + type: 'ip', + description: 'The apparent internet address of the requester.\n', + }, + { + name: 'requester', + type: 'keyword', + description: + 'The canonical user ID of the requester, or a - for unauthenticated requests.\n', + }, + { + name: 'request_id', + type: 'keyword', + description: 'A string generated by Amazon S3 to uniquely identify each request.\n', + }, + { + name: 'operation', + type: 'keyword', + description: + 'The operation listed here is declared as SOAP.operation, REST.HTTP_method.resource_type, WEBSITE.HTTP_method.resource_type, or BATCH.DELETE.OBJECT.\n', + }, + { + name: 'key', + type: 'keyword', + description: + 'The "key" part of the request, URL encoded, or "-" if the operation does not take a key parameter.\n', + }, + { + name: 'request_uri', + type: 'keyword', + description: 'The Request-URI part of the HTTP request message.\n', + }, + { + name: 'http_status', + type: 'long', + description: 'The numeric HTTP status code of the response.\n', + }, + { + name: 'error_code', + type: 'keyword', + description: 'The Amazon S3 Error Code, or "-" if no error occurred.\n', + }, + { + name: 'bytes_sent', + type: 'long', + description: + 'The number of response bytes sent, excluding HTTP protocol overhead, or "-" if zero.\n', + }, + { + name: 'object_size', + type: 'long', + description: 'The total size of the object in question.\n', + }, + { + name: 'total_time', + type: 'long', + description: + "The number of milliseconds the request was in flight from the server's perspective.\n", + }, + { + name: 'turn_around_time', + type: 'long', + description: + 'The number of milliseconds that Amazon S3 spent processing your request.\n', + }, + { + name: 'referrer', + type: 'keyword', + description: 'The value of the HTTP Referrer header, if present.\n', + }, + { + name: 'user_agent', + type: 'keyword', + description: 'The value of the HTTP User-Agent header.\n', + }, + { + name: 'version_id', + type: 'keyword', + description: + 'The version ID in the request, or "-" if the operation does not take a versionId parameter.\n', + }, + { + name: 'host_id', + type: 'keyword', + description: 'The x-amz-id-2 or Amazon S3 extended request ID.\n', + }, + { + name: 'signature_version', + type: 'keyword', + description: + 'The signature version, SigV2 or SigV4, that was used to authenticate the request or a - for unauthenticated requests.\n', + }, + { + name: 'cipher_suite', + type: 'keyword', + description: + 'The Secure Sockets Layer (SSL) cipher that was negotiated for HTTPS request or a - for HTTP.\n', + }, + { + name: 'authentication_type', + type: 'keyword', + description: + 'The type of request authentication used, AuthHeader for authentication headers, QueryString for query string (pre-signed URL) or a - for unauthenticated requests.\n', + }, + { + name: 'host_header', + type: 'keyword', + description: 'The endpoint used to connect to Amazon S3.\n', + }, + { + name: 'tls_version', + type: 'keyword', + description: + 'The Transport Layer Security (TLS) version negotiated by the client.\n', + }, + ], + }, + { + name: 'vpcflow', + type: 'group', + release: 'beta', + description: 'Fields for AWS VPC flow logs.\n', + fields: [ + { + name: 'version', + type: 'keyword', + description: + 'The VPC Flow Logs version. If you use the default format, the version is 2. If you specify a custom format, the version is 3.\n', + }, + { + name: 'account_id', + type: 'keyword', + description: 'The AWS account ID for the flow log.\n', + }, + { + name: 'interface_id', + type: 'keyword', + description: 'The ID of the network interface for which the traffic is recorded.\n', + }, + { + name: 'action', + type: 'keyword', + description: 'The action that is associated with the traffic, ACCEPT or REJECT.\n', + }, + { + name: 'log_status', + type: 'keyword', + description: 'The logging status of the flow log, OK, NODATA or SKIPDATA.\n', + }, + { + name: 'instance_id', + type: 'keyword', + description: + "The ID of the instance that's associated with network interface for which the traffic is recorded, if the instance is owned by you.\n", + }, + { + name: 'pkt_srcaddr', + type: 'ip', + description: 'The packet-level (original) source IP address of the traffic.\n', + }, + { + name: 'pkt_dstaddr', + type: 'ip', + description: + 'The packet-level (original) destination IP address for the traffic.\n', + }, + { + name: 'vpc_id', + type: 'keyword', + description: + 'The ID of the VPC that contains the network interface for which the traffic is recorded.\n', + }, + { + name: 'subnet_id', + type: 'keyword', + description: + 'The ID of the subnet that contains the network interface for which the traffic is recorded.\n', + }, + { + name: 'tcp_flags', + type: 'keyword', + description: + 'The bitmask value for the following TCP flags: 2=SYN,18=SYN-ACK,1=FIN,4=RST\n', + }, + { + name: 'type', + type: 'keyword', + description: 'The type of traffic: IPv4, IPv6, or EFA.\n', + }, + ], + }, + ], + }, + ], + }, + { + key: 'azure', + title: 'Azure', + release: 'beta', + description: 'Azure Module\n', + fields: [ + { + name: 'azure', + type: 'group', + description: '\n', + fields: [ + { + name: 'subscription_id', + type: 'keyword', + description: 'Azure subscription ID\n', + }, + { + name: 'correlation_id', + type: 'keyword', + description: 'Correlation ID\n', + }, + { + name: 'tenant_id', + type: 'keyword', + description: 'tenant ID\n', + }, + { + name: 'resource', + type: 'group', + description: 'Resource\n', + fields: [ + { + name: 'id', + type: 'keyword', + description: 'Resource ID\n', + }, + { + name: 'group', + type: 'keyword', + description: 'Resource group\n', + }, + { + name: 'provider', + type: 'keyword', + description: 'Resource type/namespace\n', + }, + { + name: 'namespace', + type: 'keyword', + description: 'Resource type/namespace\n', + }, + { + name: 'name', + type: 'keyword', + description: 'Name\n', + }, + { + name: 'authorization_rule', + type: 'keyword', + description: 'Authorization rule\n', + }, + ], + }, + { + name: 'activitylogs', + type: 'group', + release: 'beta', + description: 'Fields for Azure activity logs.\n', + fields: [ + { + name: 'identity', + type: 'group', + description: 'Identity\n', + fields: [ + { + name: 'claims_initiated_by_user', + type: 'group', + description: 'Claims initiated by user\n', + fields: [ + { + name: 'name', + type: 'keyword', + description: 'Name\n', + }, + { + name: 'givenname', + type: 'keyword', + description: 'Givenname\n', + }, + { + name: 'surname', + type: 'keyword', + description: 'Surname\n', + }, + { + name: 'fullname', + type: 'keyword', + description: 'Fullname\n', + }, + { + name: 'schema', + type: 'keyword', + description: 'Schema\n', + }, + ], + }, + { + name: 'claims.*', + type: 'object', + object_type: 'keyword', + object_type_mapping_type: '*', + description: 'Claims\n', + }, + { + name: 'authorization', + type: 'group', + description: 'Authorization\n', + fields: [ + { + name: 'scope', + type: 'keyword', + description: 'Scope\n', + }, + { + name: 'action', + type: 'keyword', + description: 'Action\n', + }, + { + name: 'evidence', + type: 'group', + description: 'Evidence\n', + fields: [ + { + name: 'role_assignment_scope', + type: 'keyword', + description: 'Role assignment scope\n', + }, + { + name: 'role_definition_id', + type: 'keyword', + description: 'Role definition ID\n', + }, + { + name: 'role', + type: 'keyword', + description: 'Role\n', + }, + { + name: 'role_assignment_id', + type: 'keyword', + description: 'Role assignment ID\n', + }, + { + name: 'principal_id', + type: 'keyword', + description: 'Principal ID\n', + }, + { + name: 'principal_type', + type: 'keyword', + description: 'Principal type\n', + }, + ], + }, + ], + }, + ], + }, + { + name: 'operation_name', + type: 'keyword', + description: 'Operation name\n', + }, + { + name: 'result_signature', + type: 'keyword', + description: 'Result signature\n', + }, + { + name: 'category', + type: 'keyword', + description: 'Category\n', + }, + { + name: 'properties', + type: 'group', + description: 'Properties\n', + fields: [ + { + name: 'service_request_id', + type: 'keyword', + description: 'Service Request Id\n', + }, + { + name: 'status_code', + type: 'keyword', + description: 'Status code\n', + }, + ], + }, + ], + }, + { + name: 'auditlogs', + type: 'group', + description: 'Fields for Azure audit logs.\n', + fields: [ + { + name: 'operation_name', + type: 'keyword', + description: 'The operation name\n', + }, + { + name: 'operation_version', + type: 'keyword', + description: 'The operation version\n', + }, + { + name: 'identity', + type: 'keyword', + description: 'Identity\n', + }, + { + name: 'tenant_id', + type: 'keyword', + description: 'Tenant ID\n', + }, + { + name: 'result_signature', + type: 'keyword', + description: 'Result signature\n', + }, + { + name: 'properties', + type: 'group', + description: 'The audit log properties\n', + fields: [ + { + name: 'result', + type: 'keyword', + description: 'Log result\n', + }, + { + name: 'activity_display_name', + type: 'keyword', + description: 'Activity display name\n', + }, + { + name: 'result_reason', + type: 'keyword', + description: 'Reason for the log result\n', + }, + { + name: 'correlation_id', + type: 'keyword', + description: 'Correlation ID\n', + }, + { + name: 'logged_by_service', + type: 'keyword', + description: 'Logged by service\n', + }, + { + name: 'operation_type', + type: 'keyword', + description: 'Operation type\n', + }, + { + name: 'id', + type: 'keyword', + description: 'ID\n', + }, + { + name: 'activity_datetime', + type: 'date', + description: 'Activity timestamp\n', + }, + { + name: 'category', + type: 'keyword', + description: 'category\n', + }, + { + name: 'target_resources.*', + type: 'group', + object_type_mapping_type: '*', + description: 'Target resources\n', + fields: [ + { + name: 'display_name', + type: 'keyword', + description: 'Display name\n', + }, + { + name: 'id', + type: 'keyword', + description: 'ID\n', + }, + { + name: 'type', + type: 'keyword', + description: 'Type\n', + }, + { + name: 'ip_address', + type: 'keyword', + description: 'ip Address\n', + }, + { + name: 'user_principal_name', + type: 'keyword', + description: 'User principal name\n', + }, + { + name: 'modified_properties.*', + type: 'group', + object_type: 'keyword', + object_type_mapping_type: '*', + description: 'Modified properties\n', + fields: [ + { + name: 'new_value', + type: 'keyword', + description: 'New value\n', + }, + { + name: 'display_name', + type: 'keyword', + description: 'Display value\n', + }, + { + name: 'old_value', + type: 'keyword', + description: 'Old value\n', + }, + ], + }, + ], + }, + { + name: 'initiated_by', + type: 'group', + description: 'Information regarding the initiator\n', + fields: [ + { + name: 'app', + type: 'group', + description: 'App\n', + fields: [ + { + name: 'servicePrincipalName', + type: 'keyword', + description: 'Service principal name\n', + }, + { + name: 'displayName', + type: 'keyword', + description: 'Display name\n', + }, + { + name: 'appId', + type: 'keyword', + description: 'App ID\n', + }, + { + name: 'servicePrincipalId', + type: 'keyword', + description: 'Service principal ID\n', + }, + ], + }, + { + name: 'user', + type: 'group', + description: 'User\n', + fields: [ + { + name: 'userPrincipalName', + type: 'keyword', + description: 'User principal name\n', + }, + { + name: 'displayName', + type: 'keyword', + description: 'Display name\n', + }, + { + name: 'id', + type: 'keyword', + description: 'ID\n', + }, + { + name: 'ipAddress', + type: 'keyword', + description: 'ip Address\n', + }, + ], + }, + ], + }, + ], + }, + ], + }, + { + name: 'signinlogs', + type: 'group', + description: 'Fields for Azure sign-in logs.\n', + fields: [ + { + name: 'operation_name', + type: 'keyword', + description: 'The operation name\n', + }, + { + name: 'operation_version', + type: 'keyword', + description: 'The operation version\n', + }, + { + name: 'tenant_id', + type: 'keyword', + description: 'Tenant ID\n', + }, + { + name: 'result_signature', + type: 'keyword', + description: 'Result signature\n', + }, + { + name: 'result_description', + type: 'keyword', + description: 'Result description\n', + }, + { + name: 'identity', + type: 'keyword', + description: 'Identity\n', + }, + { + name: 'properties', + type: 'group', + description: 'The signin log properties\n', + fields: [ + { + name: 'id', + type: 'keyword', + description: 'ID\n', + }, + { + name: 'created_at', + type: 'date', + description: 'Created date time\n', + }, + { + name: 'user_display_name', + type: 'keyword', + description: 'User display name\n', + }, + { + name: 'correlation_id', + type: 'keyword', + description: 'Correlation ID\n', + }, + { + name: 'user_principal_name', + type: 'keyword', + description: 'User principal name\n', + }, + { + name: 'user_id', + type: 'keyword', + description: 'User ID\n', + }, + { + name: 'app_id', + type: 'keyword', + description: 'App ID\n', + }, + { + name: 'app_display_name', + type: 'keyword', + description: 'App display name\n', + }, + { + name: 'ip_address', + type: 'keyword', + description: 'Ip address\n', + }, + { + name: 'client_app_used', + type: 'keyword', + description: 'Client app used\n', + }, + { + name: 'conditional_access_status', + type: 'keyword', + description: 'Conditional access status\n', + }, + { + name: 'original_request_id', + type: 'keyword', + description: 'Original request ID\n', + }, + { + name: 'is_interactive', + type: 'keyword', + description: 'Is interactive\n', + }, + { + name: 'token_issuer_name', + type: 'keyword', + description: 'Token issuer name\n', + }, + { + name: 'token_issuer_type', + type: 'keyword', + description: 'Token issuer type\n', + }, + { + name: 'processing_time_ms', + type: 'float', + description: 'Processing time in milliseconds\n', + }, + { + name: 'risk_detail', + type: 'keyword', + description: 'Risk detail\n', + }, + { + name: 'risk_level_aggregated', + type: 'keyword', + description: 'Risk level aggregated\n', + }, + { + name: 'risk_level_during_signin', + type: 'keyword', + description: 'Risk level during signIn\n', + }, + { + name: 'risk_state', + type: 'keyword', + description: 'Risk state\n', + }, + { + name: 'resource_display_name', + type: 'keyword', + description: 'Resource display name\n', + }, + { + name: 'status', + type: 'group', + description: 'Status\n', + fields: [ + { + name: 'error_code', + type: 'keyword', + description: 'Error code\n', + }, + ], + }, + { + name: 'device_detail', + type: 'group', + description: 'Status\n', + fields: [ + { + name: 'device_id', + type: 'keyword', + description: 'Device ID\n', + }, + { + name: 'operating_system', + type: 'keyword', + description: 'Operating system\n', + }, + { + name: 'browser', + type: 'keyword', + description: 'Browser\n', + }, + { + name: 'display_name', + type: 'keyword', + description: 'Display name\n', + }, + { + name: 'trust_type', + type: 'keyword', + description: 'Trust type\n', + }, + ], + }, + { + name: 'service_principal_id', + type: 'keyword', + description: 'Status\n', + }, + ], + }, + ], + }, + ], + }, + ], + }, + { + key: 'cef-module', + title: 'CEF', + description: + 'Module for receiving CEF logs over Syslog. The module adds vendor specific fields in addition to the fields the decode_cef processor provides.\n', + fields: [ + { + name: 'forcepoint', + type: 'group', + default_field: false, + description: 'Fields for Forcepoint Custom String mappings\n', + fields: [ + { + name: 'virus_id', + type: 'keyword', + description: 'Virus ID\n', + }, + ], + }, + { + name: 'checkpoint', + type: 'group', + default_field: false, + description: 'Fields for Check Point custom string mappings.\n', + fields: [ + { + name: 'app_risk', + type: 'keyword', + description: 'Application risk.', + }, + { + name: 'app_severity', + type: 'keyword', + description: 'Application threat severity.', + }, + { + name: 'app_sig_id', + type: 'keyword', + description: 'The signature ID which the application was detected by.', + }, + { + name: 'auth_method', + type: 'keyword', + description: 'Password authentication protocol used.', + }, + { + name: 'category', + type: 'keyword', + description: 'Category.', + }, + { + name: 'confidence_level', + type: 'keyword', + description: 'Confidence level determined.', + }, + { + name: 'connectivity_state', + type: 'keyword', + description: 'Connectivity state.', + }, + { + name: 'cookie', + type: 'keyword', + description: 'IKE cookie.', + }, + { + name: 'dst_phone_number', + type: 'keyword', + description: 'Destination IP-Phone.', + }, + { + name: 'email_control', + type: 'keyword', + description: 'Engine name.', + }, + { + name: 'email_id', + type: 'keyword', + description: 'Internal email ID.', + }, + { + name: 'email_recipients_num', + type: 'long', + description: 'Number of recipients.', + }, + { + name: 'email_session_id', + type: 'keyword', + description: 'Internal email session ID.', + }, + { + name: 'email_spool_id', + type: 'keyword', + description: 'Internal email spool ID.', + }, + { + name: 'email_subject', + type: 'keyword', + description: 'Email subject.', + }, + { + name: 'event_count', + type: 'long', + description: 'Number of events associated with the log.', + }, + { + name: 'file_hash', + type: 'keyword', + description: 'File hash (SHA1 or MD5).', + }, + { + name: 'frequency', + type: 'keyword', + description: 'Scan frequency.', + }, + { + name: 'icmp_type', + type: 'long', + description: 'ICMP type.', + }, + { + name: 'icmp_code', + type: 'long', + description: 'ICMP code.', + }, + { + name: 'identity_type', + type: 'keyword', + description: 'Identity type.', + }, + { + name: 'incident_extension', + type: 'keyword', + description: 'Format of original data.', + }, + { + name: 'integrity_av_invoke_type', + type: 'keyword', + description: 'Scan invoke type.', + }, + { + name: 'peer_gateway', + type: 'ip', + description: 'Main IP of the peer Security Gateway.', + }, + { + name: 'performance_impact', + type: 'keyword', + description: 'Protection performance impact.', + }, + { + name: 'protection_id', + type: 'keyword', + description: 'Protection malware ID.', + }, + { + name: 'protection_name', + type: 'keyword', + description: 'Specific signature name of the attack.', + }, + { + name: 'protection_type', + type: 'keyword', + description: 'Type of protection used to detect the attack.', + }, + { + name: 'scan_result', + type: 'keyword', + description: 'Scan result.', + }, + { + name: 'sensor_mode', + type: 'keyword', + description: 'Sensor mode.', + }, + { + name: 'severity', + type: 'keyword', + description: 'Threat severity.', + }, + { + name: 'malware_status', + type: 'keyword', + description: 'Malware status.', + }, + { + name: 'subscription_expiration', + type: 'date', + description: 'The expiration date of the subscription.', + }, + { + name: 'tcp_flags', + type: 'keyword', + description: 'TCP packet flags.', + }, + { + name: 'termination_reason', + type: 'keyword', + description: 'Termination reason.', + }, + { + name: 'update_status', + type: 'keyword', + description: 'Update status.', + }, + { + name: 'user_status', + type: 'keyword', + description: 'User response.', + }, + { + name: 'uuid', + type: 'keyword', + description: 'External ID.', + }, + { + name: 'virus_name', + type: 'keyword', + description: 'Virus name.', + }, + { + name: 'malware_name', + type: 'keyword', + description: 'Malware name.', + }, + { + name: 'malware_family', + type: 'keyword', + description: 'Malware family.', + }, + { + name: 'voip_log_type', + type: 'keyword', + description: 'VoIP log types.', + }, + ], + }, + { + name: 'cef.extensions', + type: 'group', + default_field: false, + description: 'Extra vendor-specific extensions.\n', + fields: [ + { + name: 'cp_app_risk', + type: 'keyword', + }, + { + name: 'cp_severity', + type: 'keyword', + }, + { + name: 'ifname', + type: 'keyword', + }, + { + name: 'inzone', + type: 'keyword', + }, + { + name: 'layer_uuid', + type: 'keyword', + }, + { + name: 'layer_name', + type: 'keyword', + }, + { + name: 'logid', + type: 'keyword', + }, + { + name: 'loguid', + type: 'keyword', + }, + { + name: 'match_id', + type: 'keyword', + }, + { + name: 'nat_addtnl_rulenum', + type: 'keyword', + }, + { + name: 'nat_rulenum', + type: 'keyword', + }, + { + name: 'origin', + type: 'keyword', + }, + { + name: 'originsicname', + type: 'keyword', + }, + { + name: 'outzone', + type: 'keyword', + }, + { + name: 'parent_rule', + type: 'keyword', + }, + { + name: 'product', + type: 'keyword', + }, + { + name: 'rule_action', + type: 'keyword', + }, + { + name: 'rule_uid', + type: 'keyword', + }, + { + name: 'sequencenum', + type: 'keyword', + }, + { + name: 'service_id', + type: 'keyword', + }, + { + name: 'version', + type: 'keyword', + }, + ], + }, + ], + }, + { + key: 'cisco', + title: 'Cisco', + description: 'Module for handling Cisco network device logs.\n', + fields: [ + { + name: 'cisco', + type: 'group', + description: 'Fields from Cisco logs.\n', + fields: [ + { + name: 'asa', + type: 'group', + description: 'Fields for Cisco ASA Firewall.\n', + fields: [ + { + name: 'message_id', + type: 'keyword', + description: 'The Cisco ASA message identifier.\n', + }, + { + name: 'suffix', + type: 'keyword', + example: 'session', + description: 'Optional suffix after %ASA identifier.\n', + }, + { + name: 'source_interface', + type: 'keyword', + description: 'Source interface for the flow or event.\n', + }, + { + name: 'destination_interface', + type: 'keyword', + description: 'Destination interface for the flow or event.\n', + }, + { + name: 'rule_name', + type: 'keyword', + description: 'Name of the Access Control List rule that matched this event.\n', + }, + { + name: 'source_username', + type: 'keyword', + description: 'Name of the user that is the source for this event.\n', + }, + { + name: 'destination_username', + type: 'keyword', + description: 'Name of the user that is the destination for this event.\n', + }, + { + name: 'mapped_source_ip', + type: 'ip', + description: 'The translated source IP address.\n', + }, + { + name: 'mapped_source_port', + type: 'long', + description: 'The translated source port.\n', + }, + { + name: 'mapped_destination_ip', + type: 'ip', + description: 'The translated destination IP address.\n', + }, + { + name: 'mapped_destination_port', + type: 'long', + description: 'The translated destination port.\n', + }, + { + name: 'threat_level', + type: 'keyword', + description: + 'Threat level for malware / botnet traffic. One of very-low, low, moderate, high or very-high.\n', + }, + { + name: 'threat_category', + type: 'keyword', + description: + 'Category for the malware / botnet traffic. For example: virus, botnet, trojan, etc.\n', + }, + { + name: 'connection_id', + type: 'keyword', + description: 'Unique identifier for a flow.\n', + }, + { + name: 'icmp_type', + type: 'short', + description: 'ICMP type.\n', + }, + { + name: 'icmp_code', + type: 'short', + description: 'ICMP code.\n', + }, + { + name: 'connection_type', + type: 'keyword', + default_field: false, + description: 'The VPN connection type\n', + }, + { + name: 'dap_records', + default_field: false, + type: 'keyword', + description: 'The assigned DAP records\n', + }, + ], + }, + { + name: 'ftd', + type: 'group', + description: 'Fields for Cisco Firepower Threat Defense Firewall.\n', + fields: [ + { + name: 'message_id', + type: 'keyword', + description: 'The Cisco FTD message identifier.\n', + }, + { + name: 'suffix', + type: 'keyword', + example: 'session', + description: 'Optional suffix after %FTD identifier.\n', + }, + { + name: 'source_interface', + type: 'keyword', + description: 'Source interface for the flow or event.\n', + }, + { + name: 'destination_interface', + type: 'keyword', + description: 'Destination interface for the flow or event.\n', + }, + { + name: 'rule_name', + type: 'keyword', + description: 'Name of the Access Control List rule that matched this event.\n', + }, + { + name: 'source_username', + type: 'keyword', + description: 'Name of the user that is the source for this event.\n', + }, + { + name: 'destination_username', + type: 'keyword', + description: 'Name of the user that is the destination for this event.\n', + }, + { + name: 'mapped_source_ip', + type: 'ip', + description: 'The translated source IP address. Use ECS source.nat.ip.\n', + }, + { + name: 'mapped_source_port', + type: 'long', + description: 'The translated source port. Use ECS source.nat.port.\n', + }, + { + name: 'mapped_destination_ip', + type: 'ip', + description: 'The translated destination IP address. Use ECS destination.nat.ip.\n', + }, + { + name: 'mapped_destination_port', + type: 'long', + description: 'The translated destination port. Use ECS destination.nat.port.\n', + }, + { + name: 'threat_level', + type: 'keyword', + description: + 'Threat level for malware / botnet traffic. One of very-low, low, moderate, high or very-high.\n', + }, + { + name: 'threat_category', + type: 'keyword', + description: + 'Category for the malware / botnet traffic. For example: virus, botnet, trojan, etc.\n', + }, + { + name: 'connection_id', + type: 'keyword', + description: 'Unique identifier for a flow.\n', + }, + { + name: 'icmp_type', + type: 'short', + description: 'ICMP type.\n', + }, + { + name: 'icmp_code', + type: 'short', + description: 'ICMP code.\n', + }, + { + name: 'security', + type: 'object', + description: 'Raw fields for Security Events.', + }, + { + name: 'connection_type', + type: 'keyword', + default_field: false, + description: 'The VPN connection type\n', + }, + { + name: 'dap_records', + type: 'keyword', + default_field: false, + description: 'The assigned DAP records\n', + }, + ], + }, + { + name: 'ios', + type: 'group', + description: 'Fields for Cisco IOS logs.\n', + fields: [ + { + name: 'access_list', + type: 'keyword', + description: 'Name of the IP access list.\n', + }, + { + name: 'facility', + type: 'keyword', + example: 'SEC', + description: + 'The facility to which the message refers (for example, SNMP, SYS, and so forth). A facility can be a hardware device, a protocol, or a module of the system software. It denotes the source or the cause of the system message.\n', + }, + ], + }, + ], + }, + ], + }, + { + key: 'coredns', + title: 'Coredns', + description: 'Module for handling logs produced by coredns\n', + fields: [ + { + name: 'coredns', + type: 'group', + description: 'coredns fields after normalization\n', + fields: [ + { + name: 'id', + type: 'keyword', + description: 'id of the DNS transaction\n', + }, + { + name: 'query.size', + type: 'integer', + format: 'bytes', + description: 'size of the DNS query\n', + }, + { + name: 'query.class', + type: 'keyword', + description: 'DNS query class\n', + }, + { + name: 'query.name', + type: 'keyword', + description: 'DNS query name\n', + }, + { + name: 'query.type', + type: 'keyword', + description: 'DNS query type\n', + }, + { + name: 'response.code', + type: 'keyword', + description: 'DNS response code\n', + }, + { + name: 'response.flags', + type: 'keyword', + description: 'DNS response flags\n', + }, + { + name: 'response.size', + type: 'integer', + format: 'bytes', + description: 'size of the DNS response\n', + }, + { + name: 'dnssec_ok', + type: 'boolean', + description: 'dnssec flag\n', + }, + ], + }, + ], + }, + { + key: 'envoyproxy', + title: 'Envoyproxy', + description: 'Module for handling logs produced by envoy\n', + fields: [ + { + name: 'envoyproxy', + type: 'group', + description: 'Fields from envoy proxy logs after normalization\n', + fields: [ + { + name: 'log_type', + type: 'keyword', + description: 'Envoy log type, normally ACCESS\n', + }, + { + name: 'response_flags', + type: 'keyword', + description: 'Response flags\n', + }, + { + name: 'upstream_service_time', + type: 'long', + format: 'duration', + input_format: 'nanoseconds', + description: 'Upstream service time in nanoseconds\n', + }, + { + name: 'request_id', + type: 'keyword', + description: 'ID of the request\n', + }, + { + name: 'authority', + type: 'keyword', + description: 'Envoy proxy authority field\n', + }, + { + name: 'proxy_type', + type: 'keyword', + description: 'Envoy proxy type, tcp or http\n', + }, + ], + }, + ], + }, + { + key: 'googlecloud', + title: 'Google Cloud', + description: 'Module for handling logs from Google Cloud.\n', + fields: [ + { + name: 'googlecloud', + type: 'group', + description: 'Fields from Google Cloud logs.\n', + fields: [ + { + name: 'destination.instance', + type: 'group', + description: + 'If the destination of the connection was a VM located on the same VPC, this field is populated with VM instance details. In a Shared VPC configuration, project_id corresponds to the project that owns the instance, usually the service project.\n', + fields: [ + { + name: 'project_id', + type: 'keyword', + description: 'ID of the project containing the VM.\n', + }, + { + name: 'region', + type: 'keyword', + description: 'Region of the VM.\n', + }, + { + name: 'zone', + type: 'keyword', + description: 'Zone of the VM.\n', + }, + ], + }, + { + name: 'destination.vpc', + type: 'group', + description: + 'If the destination of the connection was a VM located on the same VPC, this field is populated with VPC network details. In a Shared VPC configuration, project_id corresponds to that of the host project.\n', + fields: [ + { + name: 'project_id', + type: 'keyword', + description: 'ID of the project containing the VM.\n', + }, + { + name: 'vpc_name', + type: 'keyword', + description: 'VPC on which the VM is operating.\n', + }, + { + name: 'subnetwork_name', + type: 'keyword', + description: 'Subnetwork on which the VM is operating.\n', + }, + ], + }, + { + name: 'source.instance', + type: 'group', + description: + 'If the source of the connection was a VM located on the same VPC, this field is populated with VM instance details. In a Shared VPC configuration, project_id corresponds to the project that owns the instance, usually the service project.\n', + fields: [ + { + name: 'project_id', + type: 'keyword', + description: 'ID of the project containing the VM.\n', + }, + { + name: 'region', + type: 'keyword', + description: 'Region of the VM.\n', + }, + { + name: 'zone', + type: 'keyword', + description: 'Zone of the VM.\n', + }, + ], + }, + { + name: 'source.vpc', + type: 'group', + description: + 'If the source of the connection was a VM located on the same VPC, this field is populated with VPC network details. In a Shared VPC configuration, project_id corresponds to that of the host project.\n', + fields: [ + { + name: 'project_id', + type: 'keyword', + description: 'ID of the project containing the VM.\n', + }, + { + name: 'vpc_name', + type: 'keyword', + description: 'VPC on which the VM is operating.\n', + }, + { + name: 'subnetwork_name', + type: 'keyword', + description: 'Subnetwork on which the VM is operating.\n', + }, + ], + }, + { + name: 'audit', + type: 'group', + description: 'Fields for Google Cloud audit logs.\n', + fields: [ + { + name: 'type', + type: 'keyword', + description: 'Type property.\n', + }, + { + name: 'authentication_info', + type: 'group', + description: 'Authentication information.\n', + fields: [ + { + name: 'principal_email', + type: 'keyword', + description: + 'The email address of the authenticated user making the request.\n', + }, + { + name: 'authority_selector', + type: 'keyword', + description: + 'The authority selector specified by the requestor, if any. It is not guaranteed that the principal was allowed to use this authority.\n', + }, + ], + }, + { + name: 'authorization_info', + type: 'array', + description: 'Authorization information for the operation.\n', + fields: [ + { + name: 'permission', + type: 'keyword', + description: 'The required IAM permission.\n', + }, + { + name: 'granted', + type: 'boolean', + description: + 'Whether or not authorization for resource and permission was granted.\n', + }, + { + name: 'resource_attributes', + type: 'group', + description: 'The attributes of the resource.\n', + fields: [ + { + name: 'service', + type: 'keyword', + description: 'The name of the service.\n', + }, + { + name: 'name', + type: 'keyword', + description: 'The name of the resource.\n', + }, + { + name: 'type', + type: 'keyword', + description: 'The type of the resource.\n', + }, + ], + }, + ], + }, + { + name: 'method_name', + type: 'keyword', + description: + "The name of the service method or operation. For API calls, this should be the name of the API method. For example, 'google.datastore.v1.Datastore.RunQuery'.\n", + }, + { + name: 'num_response_items', + type: 'long', + description: + 'The number of items returned from a List or Query API method, if applicable.\n', + }, + { + name: 'request', + type: 'group', + description: 'The operation request.\n', + fields: [ + { + name: 'proto_name', + type: 'keyword', + description: 'Type property of the request.\n', + }, + { + name: 'filter', + type: 'keyword', + description: 'Filter of the request.\n', + }, + { + name: 'name', + type: 'keyword', + description: 'Name of the request.\n', + }, + { + name: 'resource_name', + type: 'keyword', + description: 'Name of the request resource.\n', + }, + ], + }, + { + name: 'request_metadata', + type: 'group', + description: 'Metadata about the request.\n', + fields: [ + { + name: 'caller_ip', + type: 'ip', + description: 'The IP address of the caller.\n', + }, + { + name: 'caller_supplied_user_agent', + type: 'keyword', + description: + 'The user agent of the caller. This information is not authenticated and should be treated accordingly.\n', + }, + ], + }, + { + name: 'resource_name', + type: 'keyword', + description: + "The resource or collection that is the target of the operation. The name is a scheme-less URI, not including the API service name. For example, 'shelves/SHELF_ID/books'.\n", + }, + { + name: 'resource_location', + type: 'group', + description: 'The location of the resource.\n', + fields: [ + { + name: 'current_locations', + type: 'keyword', + description: 'Current locations of the resource.\n', + }, + ], + }, + { + name: 'service_name', + type: 'keyword', + description: + 'The name of the API service performing the operation. For example, datastore.googleapis.com.\n', + }, + { + name: 'status', + type: 'group', + description: 'The status of the overall operation.\n', + fields: [ + { + name: 'code', + type: 'integer', + description: + 'The status code, which should be an enum value of google.rpc.Code.\n', + }, + { + name: 'message', + type: 'keyword', + description: + 'A developer-facing error message, which should be in English. Any user-facing error message should be localized and sent in the google.rpc.Status.details field, or localized by the client.\n', + }, + ], + }, + ], + }, + { + name: 'firewall', + type: 'group', + description: 'Fields for Google Cloud Firewall logs.\n', + fields: [ + { + name: 'rule_details', + type: 'group', + description: 'Description of the firewall rule that matched this connection.\n', + fields: [ + { + name: 'priority', + type: 'long', + description: 'The priority for the firewall rule.', + }, + { + name: 'action', + type: 'keyword', + description: 'Action that the rule performs on match.', + }, + { + name: 'direction', + type: 'keyword', + description: 'Direction of traffic that matches this rule.', + }, + { + name: 'reference', + type: 'keyword', + description: 'Reference to the firewall rule.', + }, + { + name: 'source_range', + type: 'keyword', + description: 'List of source ranges that the firewall rule applies to.', + }, + { + name: 'destination_range', + type: 'keyword', + description: 'List of destination ranges that the firewall applies to.', + }, + { + name: 'source_tag', + type: 'keyword', + description: 'List of all the source tags that the firewall rule applies to.\n', + }, + { + name: 'target_tag', + type: 'keyword', + description: 'List of all the target tags that the firewall rule applies to.\n', + }, + { + name: 'ip_port_info', + type: 'array', + description: 'List of ip protocols and applicable port ranges for rules.\n', + }, + { + name: 'source_service_account', + type: 'keyword', + description: + 'List of all the source service accounts that the firewall rule applies to.\n', + }, + { + name: 'target_service_account', + type: 'keyword', + description: + 'List of all the target service accounts that the firewall rule applies to.\n', + }, + ], + }, + ], + }, + { + name: 'vpcflow', + type: 'group', + description: 'Fields for Google Cloud VPC flow logs.\n', + fields: [ + { + name: 'reporter', + type: 'keyword', + description: "The side which reported the flow. Can be either 'SRC' or 'DEST'.\n", + }, + { + name: 'rtt.ms', + type: 'long', + description: + 'Latency as measured (for TCP flows only) during the time interval. This is the time elapsed between sending a SEQ and receiving a corresponding ACK and it contains the network RTT as well as the application related delay.\n', + }, + ], + }, + ], + }, + ], + }, + { + key: 'ibmmq', + title: 'ibmmq', + description: 'ibmmq Module\n', + release: 'ga', + fields: [ + { + name: 'ibmmq', + type: 'group', + description: '\n', + fields: [ + { + name: 'errorlog', + description: 'IBM MQ error logs', + type: 'group', + fields: [ + { + name: 'installation', + description: + 'This is the installation name which can be given at installation time.\nEach installation of IBM MQ on UNIX, Linux, and Windows, has a unique identifier known as an installation name. The installation name is used to associate things such as queue managers and configuration files with an installation.\n', + type: 'keyword', + }, + { + name: 'qmgr', + description: + 'Name of the queue manager. Queue managers provide queuing services to applications, and manages the queues that belong to them.\n', + type: 'keyword', + }, + { + name: 'arithinsert', + description: 'Changing content based on error.id', + type: 'keyword', + }, + { + name: 'commentinsert', + description: 'Changing content based on error.id', + type: 'keyword', + }, + { + name: 'errordescription', + description: 'Please add description', + example: 'Please add example', + type: 'text', + }, + { + name: 'explanation', + description: 'Explaines the error in more detail', + type: 'keyword', + }, + { + name: 'action', + description: 'Defines what to do when the error occurs', + type: 'keyword', + }, + { + name: 'code', + description: 'Error code.', + type: 'keyword', + }, + ], + }, + ], + }, + ], + }, + { + key: 'iptables', + title: 'iptables', + description: 'Module for handling the iptables logs.\n', + fields: [ + { + name: 'iptables', + type: 'group', + description: 'Fields from the iptables logs.\n', + fields: [ + { + name: 'ether_type', + type: 'long', + description: + 'Value of the ethernet type field identifying the network layer protocol.\n', + }, + { + name: 'flow_label', + type: 'integer', + description: 'IPv6 flow label.\n', + }, + { + name: 'fragment_flags', + type: 'keyword', + description: 'IP fragment flags. A combination of CE, DF and MF.\n', + }, + { + name: 'fragment_offset', + type: 'long', + description: 'Offset of the current IP fragment.\n', + }, + { + name: 'icmp', + type: 'group', + description: 'ICMP fields.\n', + fields: [ + { + name: 'code', + type: 'long', + description: 'ICMP code.\n', + }, + { + name: 'id', + type: 'long', + description: 'ICMP ID.\n', + }, + { + name: 'parameter', + type: 'long', + description: 'ICMP parameter.\n', + }, + { + name: 'redirect', + type: 'ip', + description: 'ICMP redirect address.\n', + }, + { + name: 'seq', + type: 'long', + description: 'ICMP sequence number.\n', + }, + { + name: 'type', + type: 'long', + description: 'ICMP type.\n', + }, + ], + }, + { + name: 'id', + type: 'long', + description: 'Packet identifier.\n', + }, + { + name: 'incomplete_bytes', + type: 'long', + description: 'Number of incomplete bytes.\n', + }, + { + name: 'input_device', + type: 'keyword', + description: 'Device that received the packet.\n', + }, + { + name: 'precedence_bits', + type: 'short', + description: 'IP precedence bits.\n', + }, + { + name: 'tos', + type: 'long', + description: 'IP Type of Service field.\n', + }, + { + name: 'length', + type: 'long', + description: 'Packet length.\n', + }, + { + name: 'output_device', + type: 'keyword', + description: 'Device that output the packet.\n', + }, + { + name: 'tcp', + type: 'group', + description: 'TCP fields.\n', + fields: [ + { + name: 'flags', + type: 'keyword', + description: 'TCP flags.\n', + }, + { + name: 'reserved_bits', + type: 'short', + description: 'TCP reserved bits.\n', + }, + { + name: 'seq', + type: 'long', + description: 'TCP sequence number.\n', + }, + { + name: 'ack', + type: 'long', + description: 'TCP Acknowledgment number.\n', + }, + { + name: 'window', + type: 'long', + description: 'Advertised TCP window size.\n', + }, + ], + }, + { + name: 'ttl', + type: 'integer', + description: 'Time To Live field.\n', + }, + { + name: 'udp', + type: 'group', + description: 'UDP fields.\n', + fields: [ + { + name: 'length', + type: 'long', + description: 'Length of the UDP header and payload.\n', + }, + ], + }, + { + name: 'ubiquiti', + type: 'group', + description: 'Fields for Ubiquiti network devices.\n', + fields: [ + { + name: 'input_zone', + type: 'keyword', + description: 'Input zone.\n', + }, + { + name: 'output_zone', + type: 'keyword', + description: 'Output zone.\n', + }, + { + name: 'rule_number', + type: 'keyword', + description: 'The rule number within the rule set.', + }, + { + name: 'rule_set', + type: 'keyword', + description: 'The rule set name.', + }, + ], + }, + ], + }, + ], + }, + { + key: 'misp', + title: 'MISP', + description: 'Module for handling threat information from MISP.\n', + fields: [ + { + name: 'misp', + type: 'group', + description: 'Fields from MISP threat information.\n', + fields: [ + { + name: 'attack_pattern', + title: 'Attack Pattern', + short: 'Fields that let you store attack patterns', + description: + 'Fields provide support for specifying information about attack patterns.\n', + type: 'group', + fields: [ + { + name: 'id', + level: 'core', + type: 'keyword', + description: 'Identifier of the threat indicator.\n', + }, + { + name: 'name', + level: 'core', + type: 'keyword', + description: 'Name of the attack pattern.\n', + }, + { + name: 'description', + level: 'extended', + type: 'text', + description: 'Description of the attack pattern.\n', + }, + { + name: 'kill_chain_phases', + level: 'extended', + type: 'keyword', + description: 'The kill chain phase(s) to which this attack pattern corresponds.\n', + }, + ], + }, + { + name: 'campaign', + title: 'Campaign', + short: 'Fields that let you store campaign information', + description: 'Fields provide support for specifying information about campaigns.\n', + type: 'group', + fields: [ + { + name: 'id', + level: 'core', + type: 'keyword', + description: 'Identifier of the campaign.\n', + }, + { + name: 'name', + level: 'core', + type: 'keyword', + description: 'Name of the campaign.\n', + }, + { + name: 'description', + level: 'extended', + type: 'text', + description: 'Description of the campaign.\n', + }, + { + name: 'aliases', + level: 'extended', + type: 'text', + description: 'Alternative names used to identify this campaign.\n', + }, + { + name: 'first_seen', + level: 'core', + type: 'date', + description: 'The time that this Campaign was first seen, in RFC3339 format.\n', + }, + { + name: 'last_seen', + level: 'core', + type: 'date', + description: 'The time that this Campaign was last seen, in RFC3339 format.\n', + }, + { + name: 'objective', + level: 'core', + type: 'keyword', + description: + "This field defines the Campaign's primary goal, objective, desired outcome, or intended effect.\n", + }, + ], + }, + { + name: 'course_of_action', + title: 'Course of Action', + short: 'Fields that let you store information about course of action.', + description: + 'A Course of Action is an action taken either to prevent an attack or to respond to an attack that is in progress.\n', + type: 'group', + fields: [ + { + name: 'id', + level: 'core', + type: 'keyword', + description: 'Identifier of the Course of Action.\n', + }, + { + name: 'name', + level: 'core', + type: 'keyword', + description: 'The name used to identify the Course of Action.\n', + }, + { + name: 'description', + level: 'extended', + type: 'text', + description: 'Description of the Course of Action.\n', + }, + ], + }, + { + name: 'identity', + title: 'Identity', + short: 'Fields that let you store information about identity.', + description: + 'Identity can represent actual individuals, organizations, or groups, as well as classes of individuals, organizations, or groups.\n', + type: 'group', + fields: [ + { + name: 'id', + level: 'core', + type: 'keyword', + description: 'Identifier of the Identity.\n', + }, + { + name: 'name', + level: 'core', + type: 'keyword', + description: 'The name used to identify the Identity.\n', + }, + { + name: 'description', + level: 'extended', + type: 'text', + description: 'Description of the Identity.\n', + }, + { + name: 'identity_class', + level: 'core', + type: 'keyword', + description: + 'The type of entity that this Identity describes, e.g., an individual or organization. Open Vocab - identity-class-ov\n', + }, + { + name: 'labels', + level: 'extended', + type: 'keyword', + description: 'The list of roles that this Identity performs.\n', + example: 'CEO\n', + }, + { + name: 'sectors', + level: 'extended', + type: 'keyword', + description: + 'The list of sectors that this Identity belongs to. Open Vocab - industry-sector-ov\n', + }, + { + name: 'contact_information', + level: 'extended', + type: 'text', + description: + 'The contact information (e-mail, phone number, etc.) for this Identity.\n', + }, + ], + }, + { + name: 'intrusion_set', + title: 'Intrusion Set', + short: 'Fields that let you store information about Intrusion Set.', + description: + 'An Intrusion Set is a grouped set of adversary behavior and resources with common properties that is believed to be orchestrated by a single organization.\n', + type: 'group', + fields: [ + { + name: 'id', + level: 'core', + type: 'keyword', + description: 'Identifier of the Intrusion Set.\n', + }, + { + name: 'name', + level: 'core', + type: 'keyword', + description: 'The name used to identify the Intrusion Set.\n', + }, + { + name: 'description', + level: 'extended', + type: 'text', + description: 'Description of the Intrusion Set.\n', + }, + { + name: 'aliases', + level: 'extended', + type: 'text', + description: 'Alternative names used to identify the Intrusion Set.\n', + }, + { + name: 'first_seen', + level: 'extended', + type: 'date', + description: + 'The time that this Intrusion Set was first seen, in RFC3339 format.\n', + }, + { + name: 'last_seen', + level: 'extended', + type: 'date', + description: 'The time that this Intrusion Set was last seen, in RFC3339 format.\n', + }, + { + name: 'goals', + level: 'extended', + type: 'text', + description: + 'The high level goals of this Intrusion Set, namely, what are they trying to do.\n', + }, + { + name: 'resource_level', + level: 'extended', + type: 'text', + description: + 'This defines the organizational level at which this Intrusion Set typically works. Open Vocab - attack-resource-level-ov\n', + }, + { + name: 'primary_motivation', + level: 'extended', + type: 'text', + description: + 'The primary reason, motivation, or purpose behind this Intrusion Set. Open Vocab - attack-motivation-ov\n', + }, + { + name: 'secondary_motivations', + level: 'extended', + type: 'text', + description: + 'The secondary reasons, motivations, or purposes behind this Intrusion Set. Open Vocab - attack-motivation-ov\n', + }, + ], + }, + { + name: 'malware', + title: 'Malware', + short: 'Fields that let you store information about Malware.', + description: + "Malware is a type of TTP that is also known as malicious code and malicious software, refers to a program that is inserted into a system, usually covertly, with the intent of compromising the confidentiality, integrity, or availability of the victim's data, applications, or operating system (OS) or of otherwise annoying or disrupting the victim.\n", + type: 'group', + fields: [ + { + name: 'id', + level: 'core', + type: 'keyword', + description: 'Identifier of the Malware.\n', + }, + { + name: 'name', + level: 'core', + type: 'keyword', + description: 'The name used to identify the Malware.\n', + }, + { + name: 'description', + level: 'extended', + type: 'text', + description: 'Description of the Malware.\n', + }, + { + name: 'labels', + level: 'core', + type: 'keyword', + description: + 'The type of malware being described. Open Vocab - malware-label-ov. adware,backdoor,bot,ddos,dropper,exploit-kit,keylogger,ransomware, remote-access-trojan,resource-exploitation,rogue-security-software,rootkit, screen-capture,spyware,trojan,virus,worm\n', + }, + { + name: 'kill_chain_phases', + format: 'string', + level: 'extended', + type: 'keyword', + description: + 'The list of kill chain phases for which this Malware instance can be used.\n', + }, + ], + }, + { + name: 'note', + title: 'Note', + short: 'Fields that let you store information about Malware.', + description: + 'A Note is a comment or note containing informative text to help explain the context of one or more STIX Objects (SDOs or SROs) or to provide additional analysis that is not contained in the original object.\n', + type: 'group', + fields: [ + { + name: 'id', + level: 'core', + type: 'keyword', + description: 'Identifier of the Note.\n', + }, + { + name: 'summary', + level: 'extended', + type: 'keyword', + description: 'A brief description used as a summary of the Note.\n', + }, + { + name: 'description', + level: 'extended', + type: 'text', + description: 'The content of the Note.\n', + }, + { + name: 'authors', + level: 'extended', + type: 'keyword', + description: 'The name of the author(s) of this Note.\n', + }, + { + name: 'object_refs', + level: 'extended', + type: 'keyword', + description: + 'The STIX Objects (SDOs and SROs) that the note is being applied to.\n', + }, + ], + }, + { + name: 'threat_indicator', + title: 'Threat Indicator', + short: 'Fields that let you store Threat Indicators', + description: + 'Fields provide support for specifying information about threat indicators, and related matching patterns.\n', + type: 'group', + fields: [ + { + name: 'labels', + level: 'core', + type: 'keyword', + description: 'list of type open-vocab that specifies the type of indicator.\n', + example: 'Domain Watchlist\n', + }, + { + name: 'id', + level: 'core', + type: 'keyword', + description: 'Identifier of the threat indicator.\n', + }, + { + name: 'version', + level: 'core', + type: 'keyword', + description: 'Version of the threat indicator.\n', + }, + { + name: 'type', + level: 'core', + type: 'keyword', + description: 'Type of the threat indicator.\n', + }, + { + name: 'description', + level: 'core', + type: 'text', + description: 'Description of the threat indicator.\n', + }, + { + name: 'feed', + level: 'core', + type: 'text', + description: 'Name of the threat feed.\n', + }, + { + name: 'valid_from', + level: 'core', + type: 'date', + description: + 'The time from which this Indicator should be considered valuable intelligence, in RFC3339 format.\n', + }, + { + name: 'valid_until', + level: 'core', + type: 'date', + description: + 'The time at which this Indicator should no longer be considered valuable intelligence. If the valid_until property is omitted, then there is no constraint on the latest time for which the indicator should be used, in RFC3339 format.\n', + }, + { + name: 'severity', + format: 'string', + level: 'core', + type: 'keyword', + description: 'Threat severity to which this indicator corresponds.\n', + example: 'high', + }, + { + name: 'confidence', + level: 'core', + type: 'keyword', + description: 'Confidence level to which this indicator corresponds.\n', + example: 'high', + }, + { + name: 'kill_chain_phases', + format: 'string', + level: 'extended', + type: 'keyword', + description: 'The kill chain phase(s) to which this indicator corresponds.\n', + }, + { + name: 'mitre_tactic', + format: 'string', + level: 'extended', + type: 'keyword', + description: 'MITRE tactics to which this indicator corresponds.\n', + example: 'Initial Access', + }, + { + name: 'mitre_technique', + format: 'string', + level: 'extended', + type: 'keyword', + description: 'MITRE techniques to which this indicator corresponds.\n', + example: 'Drive-by Compromise', + }, + { + name: 'attack_pattern', + level: 'core', + type: 'keyword', + description: + 'The attack_pattern for this indicator is a STIX Pattern as specified in STIX Version 2.0 Part 5 - STIX Patterning.\n', + example: "[destination:ip = '91.219.29.188/32']\n", + }, + { + name: 'attack_pattern_kql', + level: 'core', + type: 'keyword', + description: + 'The attack_pattern for this indicator is KQL query that matches the attack_pattern specified in the STIX Pattern format.\n', + example: 'destination.ip: "91.219.29.188/32"\n', + }, + { + name: 'negate', + level: 'core', + type: 'boolean', + description: 'When set to true, it specifies the absence of the attack_pattern.\n', + }, + { + name: 'intrusion_set', + level: 'extended', + type: 'keyword', + description: 'Name of the intrusion set if known.\n', + }, + { + name: 'campaign', + level: 'extended', + type: 'keyword', + description: 'Name of the attack campaign if known.\n', + }, + { + name: 'threat_actor', + level: 'extended', + type: 'keyword', + description: 'Name of the threat actor if known.\n', + }, + ], + }, + { + name: 'observed_data', + title: 'Observed Data', + short: 'Fields that let you store information about Observed Data.', + description: + 'Observed data conveys information that was observed on systems and networks, such as log data or network traffic, using the Cyber Observable specification.\n', + type: 'group', + fields: [ + { + name: 'id', + level: 'core', + type: 'keyword', + description: 'Identifier of the Observed Data.\n', + }, + { + name: 'first_observed', + level: 'core', + type: 'date', + description: + 'The beginning of the time window that the data was observed, in RFC3339 format.\n', + }, + { + name: 'last_observed', + level: 'core', + type: 'date', + description: + 'The end of the time window that the data was observed, in RFC3339 format.\n', + }, + { + name: 'number_observed', + level: 'core', + type: 'integer', + description: + 'The number of times the data represented in the objects property was observed. This MUST be an integer between 1 and 999,999,999 inclusive.\n', + }, + { + name: 'objects', + level: 'core', + type: 'keyword', + description: + 'A dictionary of Cyber Observable Objects that describes the single fact that was observed.\n', + }, + ], + }, + { + name: 'report', + title: 'Report', + short: 'Fields that let you store information about Report.', + description: + 'Reports are collections of threat intelligence focused on one or more topics, such as a description of a threat actor, malware, or attack technique, including context and related details.\n', + type: 'group', + fields: [ + { + name: 'id', + level: 'core', + type: 'keyword', + description: 'Identifier of the Report.\n', + }, + { + name: 'labels', + level: 'core', + type: 'keyword', + description: + 'This field is an Open Vocabulary that specifies the primary subject of this report. Open Vocab - report-label-ov. threat-report,attack-pattern,campaign,identity,indicator,malware,observed-data,threat-actor,tool,vulnerability\n', + }, + { + name: 'name', + level: 'core', + type: 'keyword', + description: 'The name used to identify the Report.\n', + }, + { + name: 'description', + level: 'extended', + type: 'text', + description: 'A description that provides more details and context about Report.\n', + }, + { + name: 'published', + level: 'extended', + type: 'date', + description: + 'The date that this report object was officially published by the creator of this report, in RFC3339 format.\n', + }, + { + name: 'object_refs', + level: 'core', + type: 'text', + description: 'Specifies the STIX Objects that are referred to by this Report.\n', + }, + ], + }, + { + name: 'threat_actor', + title: 'Threat Actor', + short: 'Fields that let you store information about Threat Actor.', + description: + 'Threat Actors are actual individuals, groups, or organizations believed to be operating with malicious intent.\n', + type: 'group', + fields: [ + { + name: 'id', + level: 'core', + type: 'keyword', + description: 'Identifier of the Threat Actor.\n', + }, + { + name: 'labels', + level: 'core', + type: 'keyword', + description: + 'This field specifies the type of threat actor. Open Vocab - threat-actor-label-ov. activist,competitor,crime-syndicate,criminal,hacker,insider-accidental,insider-disgruntled,nation-state,sensationalist,spy,terrorist\n', + }, + { + name: 'name', + level: 'core', + type: 'keyword', + description: 'The name used to identify this Threat Actor or Threat Actor group.\n', + }, + { + name: 'description', + level: 'extended', + type: 'text', + description: + 'A description that provides more details and context about the Threat Actor.\n', + }, + { + name: 'aliases', + level: 'extended', + type: 'text', + description: 'A list of other names that this Threat Actor is believed to use.\n', + }, + { + name: 'roles', + level: 'extended', + type: 'text', + description: + 'This is a list of roles the Threat Actor plays. Open Vocab - threat-actor-role-ov. agent,director,independent,sponsor,infrastructure-operator,infrastructure-architect,malware-author\n', + }, + { + name: 'goals', + level: 'extended', + type: 'text', + description: + 'The high level goals of this Threat Actor, namely, what are they trying to do.\n', + }, + { + name: 'sophistication', + level: 'extended', + type: 'text', + description: + 'The skill, specific knowledge, special training, or expertise a Threat Actor must have to perform the attack. Open Vocab - threat-actor-sophistication-ov. none,minimal,intermediate,advanced,strategic,expert,innovator\n', + }, + { + name: 'resource_level', + level: 'extended', + type: 'text', + description: + 'This defines the organizational level at which this Threat Actor typically works. Open Vocab - attack-resource-level-ov. individual,club,contest,team,organization,government\n', + }, + { + name: 'primary_motivation', + level: 'extended', + type: 'text', + description: + 'The primary reason, motivation, or purpose behind this Threat Actor. Open Vocab - attack-motivation-ov. accidental,coercion,dominance,ideology,notoriety,organizational-gain,personal-gain,personal-satisfaction,revenge,unpredictable\n', + }, + { + name: 'secondary_motivations', + level: 'extended', + type: 'text', + description: + 'The secondary reasons, motivations, or purposes behind this Threat Actor. Open Vocab - attack-motivation-ov. accidental,coercion,dominance,ideology,notoriety,organizational-gain,personal-gain,personal-satisfaction,revenge,unpredictable\n', + }, + { + name: 'personal_motivations', + level: 'extended', + type: 'text', + description: + 'The personal reasons, motivations, or purposes of the Threat Actor regardless of organizational goals. Open Vocab - attack-motivation-ov. accidental,coercion,dominance,ideology,notoriety,organizational-gain,personal-gain,personal-satisfaction,revenge,unpredictable\n', + }, + ], + }, + { + name: 'tool', + title: 'Tool', + short: 'Fields that let you store information about Tool.', + description: + 'Tools are legitimate software that can be used by threat actors to perform attacks.\n', + type: 'group', + fields: [ + { + name: 'id', + level: 'core', + type: 'keyword', + description: 'Identifier of the Tool.\n', + }, + { + name: 'labels', + level: 'core', + type: 'keyword', + description: + 'The kind(s) of tool(s) being described. Open Vocab - tool-label-ov. denial-of-service,exploitation,information-gathering,network-capture,credential-exploitation,remote-access,vulnerability-scanning\n', + }, + { + name: 'name', + level: 'core', + type: 'keyword', + description: 'The name used to identify the Tool.\n', + }, + { + name: 'description', + level: 'extended', + type: 'text', + description: + 'A description that provides more details and context about the Tool.\n', + }, + { + name: 'tool_version', + level: 'extended', + type: 'keyword', + description: 'The version identifier associated with the Tool.\n', + }, + { + name: 'kill_chain_phases', + level: 'extended', + type: 'text', + description: + 'The list of kill chain phases for which this Tool instance can be used.\n', + }, + ], + }, + { + name: 'vulnerability', + title: 'Vulnerability', + short: 'Fields that let you store information about Vulnerability.', + description: + 'A Vulnerability is a mistake in software that can be directly used by a hacker to gain access to a system or network.\n', + type: 'group', + fields: [ + { + name: 'id', + level: 'core', + type: 'keyword', + description: 'Identifier of the Vulnerability.\n', + }, + { + name: 'name', + level: 'core', + type: 'keyword', + description: 'The name used to identify the Vulnerability.\n', + }, + { + name: 'description', + level: 'extended', + type: 'text', + description: + 'A description that provides more details and context about the Vulnerability.\n', + }, + ], + }, + ], + }, + ], + }, + { + key: 'mssql', + title: 'mssql', + description: 'MS SQL Filebeat Module', + fields: [ + { + name: 'mssql', + type: 'group', + description: 'Fields from the MSSQL log files', + fields: [ + { + name: 'log', + description: 'Common log fields', + type: 'group', + fields: [ + { + name: 'origin', + description: + 'Origin of the message, usually the server but it can also be a recovery process', + type: 'keyword', + }, + ], + }, + ], + }, + ], + }, + { + key: 'netflow-module', + title: 'NetFlow', + description: + 'Module for receiving NetFlow and IPFIX flow records over UDP. The module does not add fields beyond what the netflow input provides.\n', + fields: [], + }, + { + key: 'o365', + title: 'Office 365', + description: 'Module for handling logs from Office 365.\n', + fields: [ + { + name: 'o365.audit', + type: 'group', + default_field: false, + description: 'Fields from Office 365 Management API audit logs.\n', + fields: [ + { + name: 'Actor', + type: 'array', + fields: [ + { + name: 'ID', + type: 'keyword', + }, + { + name: 'Type', + type: 'keyword', + }, + ], + }, + { + name: 'ActorContextId', + type: 'keyword', + }, + { + name: 'ActorIpAddress', + type: 'keyword', + }, + { + name: 'ActorUserId', + type: 'keyword', + }, + { + name: 'ActorYammerUserId', + type: 'keyword', + }, + { + name: 'AlertEntityId', + type: 'keyword', + }, + { + name: 'AlertId', + type: 'keyword', + }, + { + name: 'AlertLinks', + type: 'array', + }, + { + name: 'AlertType', + type: 'keyword', + }, + { + name: 'AppId', + type: 'keyword', + }, + { + name: 'ApplicationDisplayName', + type: 'keyword', + }, + { + name: 'ApplicationId', + type: 'keyword', + }, + { + name: 'AzureActiveDirectoryEventType', + type: 'keyword', + }, + { + name: 'ExchangeMetaData.*', + type: 'object', + }, + { + name: 'Category', + type: 'keyword', + }, + { + name: 'ClientAppId', + type: 'keyword', + }, + { + name: 'ClientInfoString', + type: 'keyword', + }, + { + name: 'ClientIP', + type: 'keyword', + }, + { + name: 'ClientIPAddress', + type: 'keyword', + }, + { + name: 'Comments', + type: 'text', + norms: false, + }, + { + name: 'CorrelationId', + type: 'keyword', + }, + { + name: 'CreationTime', + type: 'keyword', + }, + { + name: 'CustomUniqueId', + type: 'keyword', + }, + { + name: 'Data', + type: 'keyword', + }, + { + name: 'DataType', + type: 'keyword', + }, + { + name: 'EntityType', + type: 'keyword', + }, + { + name: 'EventData', + type: 'keyword', + }, + { + name: 'EventSource', + type: 'keyword', + }, + { + name: 'ExceptionInfo.*', + type: 'object', + }, + { + name: 'ExtendedProperties.*', + type: 'object', + }, + { + name: 'ExternalAccess', + type: 'keyword', + }, + { + name: 'GroupName', + type: 'keyword', + }, + { + name: 'Id', + type: 'keyword', + }, + { + name: 'ImplicitShare', + type: 'keyword', + }, + { + name: 'IncidentId', + type: 'keyword', + }, + { + name: 'InternalLogonType', + type: 'keyword', + }, + { + name: 'InterSystemsId', + type: 'keyword', + }, + { + name: 'IntraSystemId', + type: 'keyword', + }, + { + name: 'Item.*', + type: 'object', + }, + { + name: 'Item.*.*', + type: 'object', + }, + { + name: 'ItemName', + type: 'keyword', + }, + { + name: 'ItemType', + type: 'keyword', + }, + { + name: 'ListId', + type: 'keyword', + }, + { + name: 'ListItemUniqueId', + type: 'keyword', + }, + { + name: 'LogonError', + type: 'keyword', + }, + { + name: 'LogonType', + type: 'keyword', + }, + { + name: 'LogonUserSid', + type: 'keyword', + }, + { + name: 'MailboxGuid', + type: 'keyword', + }, + { + name: 'MailboxOwnerMasterAccountSid', + type: 'keyword', + }, + { + name: 'MailboxOwnerSid', + type: 'keyword', + }, + { + name: 'MailboxOwnerUPN', + type: 'keyword', + }, + { + name: 'Members', + type: 'array', + }, + { + name: 'Members.*', + type: 'object', + }, + { + name: 'ModifiedProperties.*.*', + type: 'object', + }, + { + name: 'Name', + type: 'keyword', + }, + { + name: 'ObjectId', + type: 'keyword', + }, + { + name: 'Operation', + type: 'keyword', + }, + { + name: 'OrganizationId', + type: 'keyword', + }, + { + name: 'OrganizationName', + type: 'keyword', + }, + { + name: 'OriginatingServer', + type: 'keyword', + }, + { + name: 'Parameters.*', + type: 'object', + }, + { + name: 'PolicyDetails', + type: 'array', + }, + { + name: 'PolicyId', + type: 'keyword', + }, + { + name: 'RecordType', + type: 'keyword', + }, + { + name: 'ResultStatus', + type: 'keyword', + }, + { + name: 'SensitiveInfoDetectionIsIncluded', + type: 'keyword', + }, + { + name: 'SharePointMetaData.*', + type: 'object', + }, + { + name: 'SessionId', + type: 'keyword', + }, + { + name: 'Severity', + type: 'keyword', + }, + { + name: 'Site', + type: 'keyword', + }, + { + name: 'SiteUrl', + type: 'keyword', + }, + { + name: 'Source', + type: 'keyword', + }, + { + name: 'SourceFileExtension', + type: 'keyword', + }, + { + name: 'SourceFileName', + type: 'keyword', + }, + { + name: 'SourceRelativeUrl', + type: 'keyword', + }, + { + name: 'Status', + type: 'keyword', + }, + { + name: 'SupportTicketId', + type: 'keyword', + }, + { + name: 'Target', + type: 'array', + fields: [ + { + name: 'ID', + type: 'keyword', + }, + { + name: 'Type', + type: 'keyword', + }, + ], + }, + { + name: 'TargetContextId', + type: 'keyword', + }, + { + name: 'TargetUserOrGroupName', + type: 'keyword', + }, + { + name: 'TargetUserOrGroupType', + type: 'keyword', + }, + { + name: 'TeamName', + type: 'keyword', + }, + { + name: 'TeamGuid', + type: 'keyword', + }, + { + name: 'UniqueSharingId', + type: 'keyword', + }, + { + name: 'UserAgent', + type: 'keyword', + }, + { + name: 'UserId', + type: 'keyword', + }, + { + name: 'UserKey', + type: 'keyword', + }, + { + name: 'UserType', + type: 'keyword', + }, + { + name: 'Version', + type: 'keyword', + }, + { + name: 'WebId', + type: 'keyword', + }, + { + name: 'Workload', + type: 'keyword', + }, + { + name: 'YammerNetworkId', + type: 'keyword', + }, + ], + }, + ], + }, + { + key: 'okta', + title: 'Okta', + description: 'Module for handling system logs from Okta.\n', + fields: [ + { + name: 'okta', + type: 'group', + default_field: false, + description: 'Fields from Okta.\n', + fields: [ + { + name: 'uuid', + title: 'UUID', + short: 'The unique identifier of the Okta LogEvent.', + description: 'The unique identifier of the Okta LogEvent.\n', + type: 'keyword', + }, + { + name: 'event_type', + title: 'Event Type', + short: 'The type of the LogEvent.', + description: 'The type of the LogEvent.\n', + type: 'keyword', + }, + { + name: 'version', + title: 'Version', + short: 'The version of the LogEvent.', + description: 'The version of the LogEvent.\n', + type: 'keyword', + }, + { + name: 'severity', + title: 'Severity', + short: 'The severity of the LogEvent.', + description: + 'The severity of the LogEvent. Must be one of DEBUG, INFO, WARN, or ERROR.\n', + type: 'keyword', + }, + { + name: 'display_message', + title: 'Display Message', + short: 'The display message of the LogEvent.', + description: 'The display message of the LogEvent.\n', + type: 'keyword', + }, + { + name: 'actor', + title: 'Actor', + short: 'Fields of the actor for the LogEvent.', + description: 'Fields that let you store information of the actor for the LogEvent.\n', + type: 'group', + fields: [ + { + name: 'id', + type: 'keyword', + description: 'Identifier of the actor.\n', + }, + { + name: 'type', + type: 'keyword', + description: 'Type of the actor.\n', + }, + { + name: 'alternate_id', + type: 'keyword', + description: 'Alternate identifier of the actor.\n', + }, + { + name: 'display_name', + type: 'keyword', + description: 'Display name of the actor.\n', + }, + ], + }, + { + name: 'client', + title: 'Client', + short: 'Fields about the client of the actor.', + description: 'Fields that let you store information about the client of the actor.\n', + type: 'group', + fields: [ + { + name: 'ip', + type: 'ip', + description: 'The IP address of the client.\n', + }, + { + name: 'user_agent', + description: 'Fields about the user agent information of the client.\n', + type: 'group', + fields: [ + { + name: 'raw_user_agent', + type: 'keyword', + description: 'The raw informaton of the user agent.\n', + }, + { + name: 'os', + type: 'keyword', + description: 'The OS informaton.\n', + }, + { + name: 'browser', + type: 'keyword', + description: 'The browser informaton of the client.\n', + }, + ], + }, + { + name: 'zone', + type: 'keyword', + description: 'The zone information of the client.\n', + }, + { + name: 'device', + type: 'keyword', + description: 'The information of the client device.\n', + }, + { + name: 'id', + type: 'keyword', + description: 'The identifier of the client.\n', + }, + ], + }, + { + name: 'outcome', + title: 'Outcome of the LogEvent.', + short: 'Fields that let you store information about the outcome.', + description: 'Fields that let you store information about the outcome.\n', + type: 'group', + fields: [ + { + name: 'reason', + type: 'keyword', + description: 'The reason of the outcome.\n', + }, + { + name: 'result', + type: 'keyword', + description: + 'The result of the outcome. Must be one of: SUCCESS, FAILURE, SKIPPED, ALLOW, DENY, CHALLENGE, UNKNOWN.\n', + }, + ], + }, + { + name: 'target', + title: 'Target', + short: 'The list of targets.', + description: 'The list of targets.\n', + type: 'array', + fields: [ + { + name: 'id', + type: 'keyword', + description: 'Identifier of the actor.\n', + }, + { + name: 'type', + type: 'keyword', + description: 'Type of the actor.\n', + }, + { + name: 'alternate_id', + type: 'keyword', + description: 'Alternate identifier of the actor.\n', + }, + { + name: 'display_name', + type: 'keyword', + description: 'Display name of the actor.\n', + }, + ], + }, + { + name: 'transaction', + title: 'Transaction', + short: 'Fields that let you store information about related transaction.', + description: 'Fields that let you store information about related transaction.\n', + type: 'group', + fields: [ + { + name: 'id', + type: 'keyword', + description: 'Identifier of the transaction.\n', + }, + { + name: 'type', + type: 'keyword', + description: 'The type of transaction. Must be one of "WEB", "JOB".\n', + }, + ], + }, + { + name: 'debug_context', + title: 'Debug Context', + short: 'Fields that let you store information about the debug context.', + description: 'Fields that let you store information about the debug context.\n', + type: 'group', + fields: [ + { + name: 'debug_data', + description: 'The debug data.\n', + type: 'group', + fields: [ + { + name: 'device_fingerprint', + type: 'keyword', + description: 'The fingerprint of the device.\n', + }, + { + name: 'request_id', + type: 'keyword', + description: 'The identifier of the request.\n', + }, + { + name: 'request_uri', + type: 'keyword', + description: 'The request URI.\n', + }, + { + name: 'threat_suspected', + type: 'keyword', + description: 'Threat suspected.\n', + }, + { + name: 'url', + type: 'keyword', + description: 'The URL.\n', + }, + ], + }, + ], + }, + { + name: 'authentication_context', + title: 'Authentication Context', + short: 'Fields that let you store information about authentication context.', + description: 'Fields that let you store information about authentication context.\n', + type: 'group', + fields: [ + { + name: 'authentication_provider', + type: 'keyword', + description: + 'The information about the authentication provider. Must be one of OKTA_AUTHENTICATION_PROVIDER, ACTIVE_DIRECTORY, LDAP, FEDERATION, SOCIAL, FACTOR_PROVIDER.\n', + }, + { + name: 'authentication_step', + type: 'integer', + description: 'The authentication step.\n', + }, + { + name: 'credential_provider', + type: 'keyword', + description: + 'The information about credential provider. Must be one of OKTA_CREDENTIAL_PROVIDER, RSA, SYMANTEC, GOOGLE, DUO, YUBIKEY.\n', + }, + { + name: 'credential_type', + type: 'keyword', + description: + 'The information about credential type. Must be one of OTP, SMS, PASSWORD, ASSERTION, IWA, EMAIL, OAUTH2, JWT, CERTIFICATE, PRE_SHARED_SYMMETRIC_KEY, OKTA_CLIENT_SESSION, DEVICE_UDID.\n', + }, + { + name: 'issuer', + description: 'The information about the issuer.\n', + type: 'array', + fields: [ + { + name: 'id', + type: 'keyword', + description: 'The identifier of the issuer.\n', + }, + { + name: 'type', + type: 'keyword', + description: 'The type of the issuer.\n', + }, + ], + }, + { + name: 'external_session_id', + type: 'keyword', + description: 'The session identifer of the external session if any.\n', + }, + { + name: 'interface', + type: 'keyword', + description: 'The interface used. e.g., Outlook, Office365, wsTrust\n', + }, + ], + }, + { + name: 'security_context', + title: 'Security Context', + short: 'Fields that let you store information about security context.', + description: 'Fields that let you store information about security context.\n', + type: 'group', + fields: [ + { + name: 'as', + type: 'group', + description: 'The autonomous system.\n', + fields: [ + { + name: 'number', + type: 'integer', + description: 'The AS number.\n', + }, + { + name: 'organization', + type: 'group', + description: 'The organization that owns the AS number.\n', + fields: [ + { + name: 'name', + type: 'keyword', + description: 'The organization name.\n', + }, + ], + }, + ], + }, + { + name: 'isp', + type: 'keyword', + description: 'The Internet Service Provider.\n', + }, + { + name: 'domain', + type: 'keyword', + description: 'The domain name.\n', + }, + { + name: 'is_proxy', + type: 'boolean', + description: 'Whether it is a proxy or not.\n', + }, + ], + }, + { + name: 'request', + title: 'Request', + short: 'Fields that let you store information about the request.', + description: + 'Fields that let you store information about the request, in the form of list of ip_chain.\n', + type: 'group', + fields: [ + { + name: 'ip_chain', + description: 'List of ip_chain objects.\n', + type: 'group', + fields: [ + { + name: 'ip', + type: 'ip', + description: 'IP address.\n', + }, + { + name: 'version', + type: 'keyword', + description: 'IP version. Must be one of V4, V6.\n', + }, + { + name: 'source', + type: 'keyword', + description: 'Source information.\n', + }, + { + name: 'geographical_context', + description: 'Geographical information.\n', + type: 'group', + fields: [ + { + name: 'city', + type: 'keyword', + description: 'The city.', + }, + { + name: 'state', + type: 'keyword', + description: 'The state.', + }, + { + name: 'postal_code', + type: 'keyword', + description: 'The postal code.', + }, + { + name: 'country', + type: 'keyword', + description: 'The country.', + }, + { + name: 'geolocation', + description: 'Geolocation information.\n', + type: 'geo_point', + }, + ], + }, + ], + }, + ], + }, + ], + }, + ], + }, + { + key: 'panw', + title: 'panw', + description: 'Module for Palo Alto Networks (PAN-OS)\n', + fields: [ + { + name: 'panw', + type: 'group', + description: 'Fields from the panw module.\n', + fields: [ + { + name: 'panos', + type: 'group', + description: 'Fields for the Palo Alto Networks PAN-OS logs.\n', + fields: [ + { + name: 'ruleset', + type: 'keyword', + description: 'Name of the rule that matched this session.\n', + }, + { + name: 'source', + type: 'group', + description: 'Fields to extend the top-level source object.\n', + fields: [ + { + name: 'zone', + type: 'keyword', + description: 'Source zone for this session.\n', + }, + { + name: 'interface', + type: 'keyword', + description: 'Source interface for this session.\n', + }, + { + name: 'nat', + type: 'group', + description: 'Post-NAT source address, if source NAT is performed.\n', + fields: [ + { + name: 'ip', + type: 'ip', + description: 'Post-NAT source IP.\n', + }, + { + name: 'port', + type: 'long', + description: 'Post-NAT source port.\n', + }, + ], + }, + ], + }, + { + name: 'destination', + type: 'group', + description: 'Fields to extend the top-level destination object.\n', + fields: [ + { + name: 'zone', + type: 'keyword', + description: 'Destination zone for this session.\n', + }, + { + name: 'interface', + type: 'keyword', + description: 'Destination interface for this session.\n', + }, + { + name: 'nat', + type: 'group', + description: 'Post-NAT destination address, if destination NAT is performed.\n', + fields: [ + { + name: 'ip', + type: 'ip', + description: 'Post-NAT destination IP.\n', + }, + { + name: 'port', + type: 'long', + description: 'Post-NAT destination port.\n', + }, + ], + }, + ], + }, + { + name: 'network', + type: 'group', + description: 'Fields to extend the top-level network object.\n', + fields: [ + { + name: 'pcap_id', + type: 'keyword', + description: 'Packet capture ID for a threat.\n', + }, + { + name: 'nat', + type: 'group', + fields: [ + { + name: 'community_id', + type: 'keyword', + description: 'Community ID flow-hash for the NAT 5-tuple.\n', + }, + ], + }, + ], + }, + { + name: 'file', + type: 'group', + description: 'Fields to extend the top-level file object.\n', + fields: [ + { + name: 'hash', + description: + 'Binary hash for a threat file sent to be analyzed by the WildFire service.\n', + type: 'keyword', + }, + ], + }, + { + name: 'url', + type: 'group', + description: 'Fields to extend the top-level url object.\n', + fields: [ + { + name: 'category', + type: 'keyword', + description: + "For threat URLs, it's the URL category. For WildFire, the verdict on the file and is either 'malicious', 'grayware', or 'benign'.\n", + }, + ], + }, + { + name: 'flow_id', + type: 'keyword', + description: 'Internal numeric identifier for each session.\n', + }, + { + name: 'sequence_number', + type: 'long', + description: + 'Log entry identifier that is incremented sequentially. Unique for each log type.\n', + }, + { + name: 'threat.resource', + type: 'keyword', + description: 'URL or file name for a threat.\n', + }, + { + name: 'threat.id', + type: 'keyword', + description: 'Palo Alto Networks identifier for the threat.\n', + }, + { + name: 'threat.name', + type: 'keyword', + description: 'Palo Alto Networks name for the threat.\n', + }, + ], + }, + ], + }, + ], + }, + { + key: 'rabbitmq', + title: 'RabbitMQ', + description: 'RabbitMQ Module\n', + fields: [ + { + name: 'rabbitmq', + type: 'group', + description: '\n', + fields: [ + { + name: 'log', + type: 'group', + description: 'RabbitMQ log files\n', + fields: [ + { + name: 'pid', + type: 'keyword', + description: 'The Erlang process id', + example: '<0.222.0>', + }, + ], + }, + ], + }, + ], + }, + { + key: 'suricata', + title: 'Suricata', + description: 'Module for handling the EVE JSON logs produced by Suricata.\n', + fields: [ + { + name: 'suricata', + type: 'group', + description: 'Fields from the Suricata EVE log file.\n', + fields: [ + { + name: 'eve', + type: 'group', + description: 'Fields exported by the EVE JSON logs\n', + fields: [ + { + name: 'event_type', + type: 'keyword', + }, + { + name: 'app_proto_orig', + type: 'keyword', + }, + { + name: 'tcp', + type: 'group', + fields: [ + { + name: 'tcp_flags', + type: 'keyword', + }, + { + name: 'psh', + type: 'boolean', + }, + { + name: 'tcp_flags_tc', + type: 'keyword', + }, + { + name: 'ack', + type: 'boolean', + }, + { + name: 'syn', + type: 'boolean', + }, + { + name: 'state', + type: 'keyword', + }, + { + name: 'tcp_flags_ts', + type: 'keyword', + }, + { + name: 'rst', + type: 'boolean', + }, + { + name: 'fin', + type: 'boolean', + }, + ], + }, + { + name: 'fileinfo', + type: 'group', + fields: [ + { + name: 'sha1', + type: 'keyword', + }, + { + name: 'filename', + type: 'alias', + path: 'file.path', + }, + { + name: 'tx_id', + type: 'long', + }, + { + name: 'state', + type: 'keyword', + }, + { + name: 'stored', + type: 'boolean', + }, + { + name: 'gaps', + type: 'boolean', + }, + { + name: 'sha256', + type: 'keyword', + }, + { + name: 'md5', + type: 'keyword', + }, + { + name: 'size', + type: 'alias', + path: 'file.size', + }, + ], + }, + { + name: 'icmp_type', + type: 'long', + }, + { + name: 'dest_port', + type: 'alias', + path: 'destination.port', + }, + { + name: 'src_port', + type: 'alias', + path: 'source.port', + }, + { + name: 'proto', + type: 'alias', + path: 'network.transport', + }, + { + name: 'pcap_cnt', + type: 'long', + }, + { + name: 'src_ip', + type: 'alias', + path: 'source.ip', + }, + { + name: 'dns', + type: 'group', + fields: [ + { + name: 'type', + type: 'keyword', + }, + { + name: 'rrtype', + type: 'keyword', + }, + { + name: 'rrname', + type: 'keyword', + }, + { + name: 'rdata', + type: 'keyword', + }, + { + name: 'tx_id', + type: 'long', + }, + { + name: 'ttl', + type: 'long', + }, + { + name: 'rcode', + type: 'keyword', + }, + { + name: 'id', + type: 'long', + }, + ], + }, + { + name: 'flow_id', + type: 'keyword', + }, + { + name: 'email', + type: 'group', + fields: [ + { + name: 'status', + type: 'keyword', + }, + ], + }, + { + name: 'dest_ip', + type: 'alias', + path: 'destination.ip', + }, + { + name: 'icmp_code', + type: 'long', + }, + { + name: 'http', + type: 'group', + fields: [ + { + name: 'status', + type: 'alias', + path: 'http.response.status_code', + }, + { + name: 'redirect', + type: 'keyword', + }, + { + name: 'http_user_agent', + type: 'alias', + path: 'user_agent.original', + }, + { + name: 'protocol', + type: 'keyword', + }, + { + name: 'http_refer', + type: 'alias', + path: 'http.request.referrer', + }, + { + name: 'url', + type: 'alias', + path: 'url.original', + }, + { + name: 'hostname', + type: 'alias', + path: 'url.domain', + }, + { + name: 'length', + type: 'alias', + path: 'http.response.body.bytes', + }, + { + name: 'http_method', + type: 'alias', + path: 'http.request.method', + }, + { + name: 'http_content_type', + type: 'keyword', + }, + ], + }, + { + name: 'timestamp', + type: 'alias', + path: '@timestamp', + }, + { + name: 'in_iface', + type: 'keyword', + }, + { + name: 'alert', + type: 'group', + fields: [ + { + name: 'category', + type: 'keyword', + }, + { + name: 'severity', + type: 'alias', + path: 'event.severity', + }, + { + name: 'rev', + type: 'long', + }, + { + name: 'gid', + type: 'long', + }, + { + name: 'signature', + type: 'keyword', + }, + { + name: 'action', + type: 'alias', + path: 'event.outcome', + }, + { + name: 'signature_id', + type: 'long', + }, + ], + }, + { + name: 'ssh', + type: 'group', + fields: [ + { + name: 'client', + type: 'group', + fields: [ + { + name: 'proto_version', + type: 'keyword', + }, + { + name: 'software_version', + type: 'keyword', + }, + ], + }, + { + name: 'server', + type: 'group', + fields: [ + { + name: 'proto_version', + type: 'keyword', + }, + { + name: 'software_version', + type: 'keyword', + }, + ], + }, + ], + }, + { + name: 'stats', + type: 'group', + fields: [ + { + name: 'capture', + type: 'group', + fields: [ + { + name: 'kernel_packets', + type: 'long', + }, + { + name: 'kernel_drops', + type: 'long', + }, + { + name: 'kernel_ifdrops', + type: 'long', + }, + ], + }, + { + name: 'uptime', + type: 'long', + }, + { + name: 'detect', + type: 'group', + fields: [ + { + name: 'alert', + type: 'long', + }, + ], + }, + { + name: 'http', + type: 'group', + fields: [ + { + name: 'memcap', + type: 'long', + }, + { + name: 'memuse', + type: 'long', + }, + ], + }, + { + name: 'file_store', + type: 'group', + fields: [ + { + name: 'open_files', + type: 'long', + }, + ], + }, + { + name: 'defrag', + type: 'group', + fields: [ + { + name: 'max_frag_hits', + type: 'long', + }, + { + name: 'ipv4', + type: 'group', + fields: [ + { + name: 'timeouts', + type: 'long', + }, + { + name: 'fragments', + type: 'long', + }, + { + name: 'reassembled', + type: 'long', + }, + ], + }, + { + name: 'ipv6', + type: 'group', + fields: [ + { + name: 'timeouts', + type: 'long', + }, + { + name: 'fragments', + type: 'long', + }, + { + name: 'reassembled', + type: 'long', + }, + ], + }, + ], + }, + { + name: 'flow', + type: 'group', + fields: [ + { + name: 'tcp_reuse', + type: 'long', + }, + { + name: 'udp', + type: 'long', + }, + { + name: 'memcap', + type: 'long', + }, + { + name: 'emerg_mode_entered', + type: 'long', + }, + { + name: 'emerg_mode_over', + type: 'long', + }, + { + name: 'tcp', + type: 'long', + }, + { + name: 'icmpv6', + type: 'long', + }, + { + name: 'icmpv4', + type: 'long', + }, + { + name: 'spare', + type: 'long', + }, + { + name: 'memuse', + type: 'long', + }, + ], + }, + { + name: 'tcp', + type: 'group', + fields: [ + { + name: 'pseudo_failed', + type: 'long', + }, + { + name: 'ssn_memcap_drop', + type: 'long', + }, + { + name: 'insert_data_overlap_fail', + type: 'long', + }, + { + name: 'sessions', + type: 'long', + }, + { + name: 'pseudo', + type: 'long', + }, + { + name: 'synack', + type: 'long', + }, + { + name: 'insert_data_normal_fail', + type: 'long', + }, + { + name: 'syn', + type: 'long', + }, + { + name: 'memuse', + type: 'long', + }, + { + name: 'invalid_checksum', + type: 'long', + }, + { + name: 'segment_memcap_drop', + type: 'long', + }, + { + name: 'overlap', + type: 'long', + }, + { + name: 'insert_list_fail', + type: 'long', + }, + { + name: 'rst', + type: 'long', + }, + { + name: 'stream_depth_reached', + type: 'long', + }, + { + name: 'reassembly_memuse', + type: 'long', + }, + { + name: 'reassembly_gap', + type: 'long', + }, + { + name: 'overlap_diff_data', + type: 'long', + }, + { + name: 'no_flow', type: 'long', }, + ], + }, + { + name: 'decoder', + type: 'group', + fields: [ { - name: 'segment_memcap_drop', + name: 'avg_pkt_size', type: 'long', }, { - name: 'overlap', + name: 'bytes', type: 'long', }, { - name: 'insert_list_fail', + name: 'tcp', type: 'long', }, { - name: 'rst', + name: 'raw', type: 'long', }, { - name: 'stream_depth_reached', + name: 'ppp', type: 'long', }, { - name: 'reassembly_memuse', + name: 'vlan_qinq', type: 'long', }, { - name: 'reassembly_gap', + name: 'null', type: 'long', }, { - name: 'overlap_diff_data', + name: 'ltnull', + type: 'group', + fields: [ + { + name: 'unsupported_type', + type: 'long', + }, + { + name: 'pkt_too_small', + type: 'long', + }, + ], + }, + { + name: 'invalid', + type: 'long', + }, + { + name: 'gre', + type: 'long', + }, + { + name: 'ipv4', + type: 'long', + }, + { + name: 'ipv6', + type: 'long', + }, + { + name: 'pkts', + type: 'long', + }, + { + name: 'ipv6_in_ipv6', + type: 'long', + }, + { + name: 'ipraw', + type: 'group', + fields: [ + { + name: 'invalid_ip_version', + type: 'long', + }, + ], + }, + { + name: 'pppoe', + type: 'long', + }, + { + name: 'udp', + type: 'long', + }, + { + name: 'dce', + type: 'group', + fields: [ + { + name: 'pkt_too_small', + type: 'long', + }, + ], + }, + { + name: 'vlan', + type: 'long', + }, + { + name: 'sctp', + type: 'long', + }, + { + name: 'max_pkt_size', + type: 'long', + }, + { + name: 'teredo', + type: 'long', + }, + { + name: 'mpls', + type: 'long', + }, + { + name: 'sll', + type: 'long', + }, + { + name: 'icmpv6', + type: 'long', + }, + { + name: 'icmpv4', + type: 'long', + }, + { + name: 'erspan', + type: 'long', + }, + { + name: 'ethernet', + type: 'long', + }, + { + name: 'ipv4_in_ipv6', + type: 'long', + }, + { + name: 'ieee8021ah', + type: 'long', + }, + ], + }, + { + name: 'dns', + type: 'group', + fields: [ + { + name: 'memcap_global', + type: 'long', + }, + { + name: 'memcap_state', + type: 'long', + }, + { + name: 'memuse', + type: 'long', + }, + ], + }, + { + name: 'flow_mgr', + type: 'group', + fields: [ + { + name: 'rows_busy', + type: 'long', + }, + { + name: 'flows_timeout', + type: 'long', + }, + { + name: 'flows_notimeout', + type: 'long', + }, + { + name: 'rows_skipped', + type: 'long', + }, + { + name: 'closed_pruned', + type: 'long', + }, + { + name: 'new_pruned', + type: 'long', + }, + { + name: 'flows_removed', + type: 'long', + }, + { + name: 'bypassed_pruned', + type: 'long', + }, + { + name: 'est_pruned', + type: 'long', + }, + { + name: 'flows_timeout_inuse', + type: 'long', + }, + { + name: 'flows_checked', + type: 'long', + }, + { + name: 'rows_maxlen', + type: 'long', + }, + { + name: 'rows_checked', + type: 'long', + }, + { + name: 'rows_empty', type: 'long', }, + ], + }, + { + name: 'app_layer', + type: 'group', + fields: [ + { + name: 'flow', + type: 'group', + fields: [ + { + name: 'tls', + type: 'long', + }, + { + name: 'ftp', + type: 'long', + }, + { + name: 'http', + type: 'long', + }, + { + name: 'failed_udp', + type: 'long', + }, + { + name: 'dns_udp', + type: 'long', + }, + { + name: 'dns_tcp', + type: 'long', + }, + { + name: 'smtp', + type: 'long', + }, + { + name: 'failed_tcp', + type: 'long', + }, + { + name: 'msn', + type: 'long', + }, + { + name: 'ssh', + type: 'long', + }, + { + name: 'imap', + type: 'long', + }, + { + name: 'dcerpc_udp', + type: 'long', + }, + { + name: 'dcerpc_tcp', + type: 'long', + }, + { + name: 'smb', + type: 'long', + }, + ], + }, { - name: 'no_flow', - type: 'long', + name: 'tx', + type: 'group', + fields: [ + { + name: 'tls', + type: 'long', + }, + { + name: 'ftp', + type: 'long', + }, + { + name: 'http', + type: 'long', + }, + { + name: 'dns_udp', + type: 'long', + }, + { + name: 'dns_tcp', + type: 'long', + }, + { + name: 'smtp', + type: 'long', + }, + { + name: 'ssh', + type: 'long', + }, + { + name: 'dcerpc_udp', + type: 'long', + }, + { + name: 'dcerpc_tcp', + type: 'long', + }, + { + name: 'smb', + type: 'long', + }, + ], }, ], }, + ], + }, + { + name: 'tls', + type: 'group', + fields: [ + { + name: 'notbefore', + type: 'date', + }, + { + name: 'issuerdn', + type: 'keyword', + }, + { + name: 'sni', + type: 'keyword', + }, + { + name: 'version', + type: 'keyword', + }, + { + name: 'session_resumed', + type: 'boolean', + }, + { + name: 'fingerprint', + type: 'keyword', + }, + { + name: 'serial', + type: 'keyword', + }, + { + name: 'notafter', + type: 'date', + }, + { + name: 'subject', + type: 'keyword', + }, + ], + }, + { + name: 'app_proto_ts', + type: 'keyword', + }, + { + name: 'flow', + type: 'group', + fields: [ + { + name: 'bytes_toclient', + type: 'alias', + path: 'destination.bytes', + }, + { + name: 'start', + type: 'alias', + path: 'event.start', + }, + { + name: 'pkts_toclient', + type: 'alias', + path: 'destination.packets', + }, + { + name: 'age', + type: 'long', + }, + { + name: 'state', + type: 'keyword', + }, + { + name: 'bytes_toserver', + type: 'alias', + path: 'source.bytes', + }, + { + name: 'reason', + type: 'keyword', + }, + { + name: 'pkts_toserver', + type: 'alias', + path: 'source.packets', + }, + { + name: 'end', + type: 'date', + }, + { + name: 'alerted', + type: 'boolean', + }, + ], + }, + { + name: 'app_proto', + type: 'alias', + path: 'network.protocol', + }, + { + name: 'tx_id', + type: 'long', + }, + { + name: 'app_proto_tc', + type: 'keyword', + }, + { + name: 'smtp', + type: 'group', + fields: [ + { + name: 'rcpt_to', + type: 'keyword', + }, + { + name: 'mail_from', + type: 'keyword', + }, + { + name: 'helo', + type: 'keyword', + }, + ], + }, + { + name: 'app_proto_expected', + type: 'keyword', + }, + { + name: 'flags', + type: 'group', + fields: [], + }, + ], + }, + ], + }, + ], + }, + { + key: 'zeek', + title: 'Zeek', + description: 'Module for handling logs produced by Zeek/Bro\n', + fields: [ + { + name: 'zeek', + type: 'group', + description: 'Fields from Zeek/Bro logs after normalization\n', + fields: [ + { + name: 'session_id', + type: 'keyword', + description: 'A unique identifier of the session\n', + }, + { + name: 'capture_loss', + type: 'group', + description: 'Fields exported by the Zeek capture_loss log\n', + fields: [ + { + name: 'ts_delta', + type: 'integer', + description: 'The time delay between this measurement and the last.\n', + }, + { + name: 'peer', + type: 'keyword', + description: + 'In the event that there are multiple Bro instances logging to the same host, this distinguishes each peer with its individual name.\n', + }, + { + name: 'gaps', + type: 'integer', + description: 'Number of missed ACKs from the previous measurement interval.\n', + }, + { + name: 'acks', + type: 'integer', + description: 'Total number of ACKs seen in the previous measurement interval.\n', + }, + { + name: 'percent_lost', + type: 'double', + description: "Percentage of ACKs seen where the data being ACKed wasn't seen.\n", + }, + ], + }, + { + name: 'connection', + type: 'group', + default_field: false, + description: 'Fields exported by the Zeek Connection log\n', + fields: [ + { + name: 'local_orig', + type: 'boolean', + description: 'Indicates whether the session is originated locally.\n', + }, + { + name: 'local_resp', + type: 'boolean', + description: 'Indicates whether the session is responded locally.\n', + }, + { + name: 'missed_bytes', + type: 'long', + description: 'Missed bytes for the session.\n', + }, + { + name: 'state', + type: 'keyword', + description: 'Code indicating the state of the session.\n', + }, + { + name: 'state_message', + type: 'keyword', + description: 'The state of the session.\n', + }, + { + name: 'icmp', + type: 'group', + fields: [ + { + name: 'type', + type: 'integer', + description: 'ICMP message type.\n', + }, + { + name: 'code', + type: 'integer', + description: 'ICMP message code.\n', + }, + ], + }, + { + name: 'history', + type: 'keyword', + description: 'Flags indicating the history of the session.\n', + }, + { + name: 'vlan', + type: 'integer', + description: 'VLAN identifier.\n', + }, + { + name: 'inner_vlan', + type: 'integer', + description: 'VLAN identifier.\n', + }, + ], + }, + { + name: 'dce_rpc', + type: 'group', + default_field: false, + description: 'Fields exported by the Zeek DCE_RPC log\n', + fields: [ + { + name: 'rtt', + type: 'integer', + description: + "Round trip time from the request to the response. If either the request or response wasn't seen, this will be null.\n", + }, + { + name: 'named_pipe', + type: 'keyword', + description: 'Remote pipe name.\n', + }, + { + name: 'endpoint', + type: 'keyword', + description: 'Endpoint name looked up from the uuid.\n', + }, + { + name: 'operation', + type: 'keyword', + description: 'Operation seen in the call.\n', + }, + ], + }, + { + name: 'dhcp', + type: 'group', + default_field: false, + description: 'Fields exported by the Zeek DHCP log\n', + fields: [ + { + name: 'domain', + type: 'keyword', + description: 'Domain given by the server in option 15.\n', + }, + { + name: 'duration', + type: 'double', + description: + 'Duration of the DHCP session representing the time from the first\nmessage to the last, in seconds.\n', + }, + { + name: 'hostname', + type: 'keyword', + description: 'Name given by client in Hostname option 12.\n', + }, + { + name: 'client_fqdn', + type: 'keyword', + description: 'FQDN given by client in Client FQDN option 81.\n', + }, + { + name: 'lease_time', + type: 'integer', + description: 'IP address lease interval in seconds.\n', + }, + { + name: 'address', + type: 'group', + description: 'Addresses seen in this DHCP exchange.\n', + fields: [ + { + name: 'assigned', + type: 'ip', + description: 'IP address assigned by the server.\n', + }, + { + name: 'client', + type: 'ip', + description: + 'IP address of the client. If a transaction is only a client sending\nINFORM messages then there is no lease information exchanged so this\nis helpful to know who sent the messages. Getting an address in this\nfield does require that the client sources at least one DHCP message\nusing a non-broadcast address.\n', + }, + { + name: 'mac', + type: 'keyword', + description: "Client's hardware address.\n", + }, + { + name: 'requested', + type: 'ip', + description: 'IP address requested by the client.\n', + }, + { + name: 'server', + type: 'ip', + description: 'IP address of the DHCP server.\n', + }, + ], + }, + { + name: 'msg', + type: 'group', + fields: [ + { + name: 'types', + type: 'keyword', + description: 'List of DHCP message types seen in this exchange.\n', + }, + { + name: 'origin', + type: 'ip', + description: + '(present if policy/protocols/dhcp/msg-orig.bro is loaded)\nThe address that originated each message from the msg.types field.\n', + }, + { + name: 'client', + type: 'keyword', + description: + 'Message typically accompanied with a DHCP_DECLINE so the client can\ntell the server why it rejected an address.\n', + }, + { + name: 'server', + type: 'keyword', + description: + 'Message typically accompanied with a DHCP_NAK to let the client know\nwhy it rejected the request.\n', + }, + ], + }, + { + name: 'software', + type: 'group', + fields: [ + { + name: 'client', + type: 'keyword', + description: + '(present if policy/protocols/dhcp/software.bro is loaded)\nSoftware reported by the client in the vendor_class option.\n', + }, + { + name: 'server', + type: 'keyword', + description: + '(present if policy/protocols/dhcp/software.bro is loaded)\nSoftware reported by the client in the vendor_class option.\n', + }, + ], + }, + { + name: 'id', + type: 'group', + fields: [ + { + name: 'circuit', + type: 'keyword', + description: + '(present if policy/protocols/dhcp/sub-opts.bro is loaded)\nAdded by DHCP relay agents which terminate switched or permanent\ncircuits. It encodes an agent-local identifier of the circuit from\nwhich a DHCP client-to-server packet was received. Typically it\nshould represent a router or switch interface number.\n', + }, + { + name: 'remote_agent', + type: 'keyword', + description: + '(present if policy/protocols/dhcp/sub-opts.bro is loaded)\nA globally unique identifier added by relay agents to identify the\nremote host end of the circuit.\n', + }, + { + name: 'subscriber', + type: 'keyword', + description: + "(present if policy/protocols/dhcp/sub-opts.bro is loaded)\nThe subscriber ID is a value independent of the physical network\nconfiguration so that a customer's DHCP configuration can be given\nto them correctly no matter where they are physically connected.\n", + }, + ], + }, + ], + }, + { + name: 'dnp3', + type: 'group', + default_field: false, + description: 'Fields exported by the Zeek SSH log\n', + fields: [ + { + name: 'function', + type: 'group', + fields: [ + { + name: 'request', + type: 'keyword', + description: 'The name of the function message in the request.\n', + }, + { + name: 'reply', + type: 'keyword', + description: 'The name of the function message in the reply.\n', + }, + ], + }, + { + name: 'id', + type: 'integer', + description: "The response's internal indication number.\n", + }, + ], + }, + { + name: 'dns', + type: 'group', + description: 'Fields exported by the Zeek DNS log\n', + fields: [ + { + name: 'trans_id', + type: 'keyword', + description: 'DNS transaction identifier.\n', + }, + { + name: 'rtt', + type: 'double', + description: 'Round trip time for the query and response.\n', + }, + { + name: 'query', + type: 'keyword', + description: 'The domain name that is the subject of the DNS query.\n', + }, + { + name: 'qclass', + type: 'long', + description: 'The QCLASS value specifying the class of the query.\n', + }, + { + name: 'qclass_name', + type: 'keyword', + description: 'A descriptive name for the class of the query.\n', + }, + { + name: 'qtype', + type: 'long', + description: 'A QTYPE value specifying the type of the query.\n', + }, + { + name: 'qtype_name', + type: 'keyword', + description: 'A descriptive name for the type of the query.\n', + }, + { + name: 'rcode', + type: 'long', + description: 'The response code value in DNS response messages.\n', + }, + { + name: 'rcode_name', + type: 'keyword', + description: 'A descriptive name for the response code value.\n', + }, + { + name: 'AA', + type: 'boolean', + description: + 'The Authoritative Answer bit for response messages specifies that the responding\nname server is an authority for the domain name in the question section.\n', + }, + { + name: 'TC', + type: 'boolean', + description: 'The Truncation bit specifies that the message was truncated.\n', + }, + { + name: 'RD', + type: 'boolean', + description: + 'The Recursion Desired bit in a request message indicates that the client\nwants recursive service for this query.\n', + }, + { + name: 'RA', + type: 'boolean', + description: + 'The Recursion Available bit in a response message indicates that the name\nserver supports recursive queries.\n', + }, + { + name: 'answers', + type: 'keyword', + description: 'The set of resource descriptions in the query answer.\n', + }, + { + name: 'TTLs', + type: 'double', + description: + 'The caching intervals of the associated RRs described by the answers field.\n', + }, + { + name: 'rejected', + type: 'boolean', + description: 'Indicates whether the DNS query was rejected by the server.\n', + }, + { + name: 'total_answers', + type: 'integer', + description: 'The total number of resource records in the reply.\n', + }, + { + name: 'total_replies', + type: 'integer', + description: 'The total number of resource records in the reply message.\n', + }, + { + name: 'saw_query', + type: 'boolean', + description: 'Whether the full DNS query has been seen.\n', + }, + { + name: 'saw_reply', + type: 'boolean', + description: 'Whether the full DNS reply has been seen.\n', + }, + ], + }, + { + name: 'dpd', + type: 'group', + default_field: false, + description: 'Fields exported by the Zeek DPD log\n', + fields: [ + { + name: 'analyzer', + type: 'keyword', + description: 'The analyzer that generated the violation.\n', + }, + { + name: 'failure_reason', + type: 'keyword', + description: 'The textual reason for the analysis failure.\n', + }, + { + name: 'packet_segment', + type: 'keyword', + description: + '(present if policy/frameworks/dpd/packet-segment-logging.bro is loaded)\nA chunk of the payload that most likely resulted in the protocol violation.\n', + }, + ], + }, + { + name: 'files', + type: 'group', + description: 'Fields exported by the Zeek Files log.\n', + fields: [ + { + name: 'fuid', + type: 'keyword', + description: 'A file unique identifier.\n', + }, + { + name: 'tx_host', + type: 'ip', + description: 'The host that transferred the file.\n', + }, + { + name: 'rx_host', + type: 'ip', + description: 'The host that received the file.\n', + }, + { + name: 'session_ids', + type: 'keyword', + description: 'The sessions that have this file.\n', + }, + { + name: 'source', + type: 'keyword', + description: + 'An identification of the source of the file data. E.g. it may be a network protocol\nover which it was transferred, or a local file path which was read, or some other\ninput source.\n', + }, + { + name: 'depth', + type: 'long', + description: + 'A value to represent the depth of this file in relation to its source. In SMTP, it\nis the depth of the MIME attachment on the message. In HTTP, it is the depth of the\nrequest within the TCP connection.\n', + }, + { + name: 'analyzers', + type: 'keyword', + description: 'A set of analysis types done during the file analysis.\n', + }, + { + name: 'mime_type', + type: 'keyword', + description: 'Mime type of the file.\n', + }, + { + name: 'filename', + type: 'keyword', + description: 'Name of the file if available.\n', + }, + { + name: 'local_orig', + type: 'boolean', + description: + 'If the source of this file is a network connection, this field indicates if the data\noriginated from the local network or not.\n', + }, + { + name: 'is_orig', + type: 'boolean', + description: + 'If the source of this file is a network connection, this field indicates if the file is\nbeing sent by the originator of the connection or the responder.\n', + }, + { + name: 'duration', + type: 'double', + description: + 'The duration the file was analyzed for. Not the duration of the session.\n', + }, + { + name: 'seen_bytes', + type: 'long', + description: 'Number of bytes provided to the file analysis engine for the file.\n', + }, + { + name: 'total_bytes', + type: 'long', + description: 'Total number of bytes that are supposed to comprise the full file.\n', + }, + { + name: 'missing_bytes', + type: 'long', + description: + 'The number of bytes in the file stream that were completely missed during the process\nof analysis.\n', + }, + { + name: 'overflow_bytes', + type: 'long', + description: + "The number of bytes in the file stream that were not delivered to stream file analyzers.\nThis could be overlapping bytes or bytes that couldn't be reassembled.\n", + }, + { + name: 'timedout', + type: 'boolean', + description: 'Whether the file analysis timed out at least once for the file.\n', + }, + { + name: 'parent_fuid', + type: 'keyword', + description: + 'Identifier associated with a container file from which this one was extracted as part of\nthe file analysis.\n', + }, + { + name: 'md5', + type: 'keyword', + description: 'An MD5 digest of the file contents.\n', + }, + { + name: 'sha1', + type: 'keyword', + description: 'A SHA1 digest of the file contents.\n', + }, + { + name: 'sha256', + type: 'keyword', + description: 'A SHA256 digest of the file contents.\n', + }, + { + name: 'extracted', + type: 'keyword', + description: 'Local filename of extracted file.\n', + }, + { + name: 'extracted_cutoff', + type: 'boolean', + description: + 'Indicate whether the file being extracted was cut off hence not extracted completely.\n', + }, + { + name: 'extracted_size', + type: 'long', + description: 'The number of bytes extracted to disk.\n', + }, + { + name: 'entropy', + type: 'double', + description: 'The information density of the contents of the file.\n', + }, + ], + }, + { + name: 'ftp', + type: 'group', + default_field: false, + description: 'Fields exported by the Zeek FTP log\n', + fields: [ + { + name: 'user', + type: 'keyword', + description: 'User name for the current FTP session.\n', + }, + { + name: 'password', + type: 'keyword', + description: 'Password for the current FTP session if captured.\n', + }, + { + name: 'command', + type: 'keyword', + description: 'Command given by the client.\n', + }, + { + name: 'arg', + type: 'keyword', + description: 'Argument for the command if one is given.\n', + }, + { + name: 'file', + type: 'group', + fields: [ + { + name: 'size', + type: 'long', + description: 'Size of the file if the command indicates a file transfer.\n', + }, { - name: 'decoder', - type: 'group', - fields: [ - { - name: 'avg_pkt_size', - type: 'long', - }, - { - name: 'bytes', - type: 'long', - }, - { - name: 'tcp', - type: 'long', - }, - { - name: 'raw', - type: 'long', - }, - { - name: 'ppp', - type: 'long', - }, - { - name: 'vlan_qinq', - type: 'long', - }, - { - name: 'null', - type: 'long', - }, - { - name: 'ltnull', - type: 'group', - fields: [ - { - name: 'unsupported_type', - type: 'long', - }, - { - name: 'pkt_too_small', - type: 'long', - }, - ], - }, - { - name: 'invalid', - type: 'long', - }, - { - name: 'gre', - type: 'long', - }, - { - name: 'ipv4', - type: 'long', - }, - { - name: 'ipv6', - type: 'long', - }, - { - name: 'pkts', - type: 'long', - }, - { - name: 'ipv6_in_ipv6', - type: 'long', - }, - { - name: 'ipraw', - type: 'group', - fields: [ - { - name: 'invalid_ip_version', - type: 'long', - }, - ], - }, - { - name: 'pppoe', - type: 'long', - }, - { - name: 'udp', - type: 'long', - }, - { - name: 'dce', - type: 'group', - fields: [ - { - name: 'pkt_too_small', - type: 'long', - }, - ], - }, - { - name: 'vlan', - type: 'long', - }, - { - name: 'sctp', - type: 'long', - }, - { - name: 'max_pkt_size', - type: 'long', - }, - { - name: 'teredo', - type: 'long', - }, - { - name: 'mpls', - type: 'long', - }, - { - name: 'sll', - type: 'long', - }, - { - name: 'icmpv6', - type: 'long', - }, - { - name: 'icmpv4', - type: 'long', - }, - { - name: 'erspan', - type: 'long', - }, - { - name: 'ethernet', - type: 'long', - }, - { - name: 'ipv4_in_ipv6', - type: 'long', - }, - { - name: 'ieee8021ah', - type: 'long', - }, - ], + name: 'mime_type', + type: 'keyword', + description: 'Sniffed mime type of file.\n', + }, + { + name: 'fuid', + type: 'keyword', + description: + '(present if base/protocols/ftp/files.bro is loaded)\nFile unique ID.\n', + }, + ], + }, + { + name: 'reply', + type: 'group', + fields: [ + { + name: 'code', + type: 'integer', + description: 'Reply code from the server in response to the command.\n', }, { - name: 'dns', - type: 'group', - fields: [ - { - name: 'memcap_global', - type: 'long', - }, - { - name: 'memcap_state', - type: 'long', - }, - { - name: 'memuse', - type: 'long', - }, - ], + name: 'msg', + type: 'keyword', + description: 'Reply message from the server in response to the command.\n', }, + ], + }, + { + name: 'data_channel', + type: 'group', + description: 'Expected FTP data channel.\n', + fields: [ { - name: 'flow_mgr', - type: 'group', - fields: [ - { - name: 'rows_busy', - type: 'long', - }, - { - name: 'flows_timeout', - type: 'long', - }, - { - name: 'flows_notimeout', - type: 'long', - }, - { - name: 'rows_skipped', - type: 'long', - }, - { - name: 'closed_pruned', - type: 'long', - }, - { - name: 'new_pruned', - type: 'long', - }, - { - name: 'flows_removed', - type: 'long', - }, - { - name: 'bypassed_pruned', - type: 'long', - }, - { - name: 'est_pruned', - type: 'long', - }, - { - name: 'flows_timeout_inuse', - type: 'long', - }, - { - name: 'flows_checked', - type: 'long', - }, - { - name: 'rows_maxlen', - type: 'long', - }, - { - name: 'rows_checked', - type: 'long', - }, - { - name: 'rows_empty', - type: 'long', - }, - ], + name: 'passive', + type: 'boolean', + description: 'Whether PASV mode is toggled for control channel.\n', }, { - name: 'app_layer', - type: 'group', - fields: [ - { - name: 'flow', - type: 'group', - fields: [ - { - name: 'tls', - type: 'long', - }, - { - name: 'ftp', - type: 'long', - }, - { - name: 'http', - type: 'long', - }, - { - name: 'failed_udp', - type: 'long', - }, - { - name: 'dns_udp', - type: 'long', - }, - { - name: 'dns_tcp', - type: 'long', - }, - { - name: 'smtp', - type: 'long', - }, - { - name: 'failed_tcp', - type: 'long', - }, - { - name: 'msn', - type: 'long', - }, - { - name: 'ssh', - type: 'long', - }, - { - name: 'imap', - type: 'long', - }, - { - name: 'dcerpc_udp', - type: 'long', - }, - { - name: 'dcerpc_tcp', - type: 'long', - }, - { - name: 'smb', - type: 'long', - }, - ], - }, - { - name: 'tx', - type: 'group', - fields: [ - { - name: 'tls', - type: 'long', - }, - { - name: 'ftp', - type: 'long', - }, - { - name: 'http', - type: 'long', - }, - { - name: 'dns_udp', - type: 'long', - }, - { - name: 'dns_tcp', - type: 'long', - }, - { - name: 'smtp', - type: 'long', - }, - { - name: 'ssh', - type: 'long', - }, - { - name: 'dcerpc_udp', - type: 'long', - }, - { - name: 'dcerpc_tcp', - type: 'long', - }, - { - name: 'smb', - type: 'long', - }, - ], - }, - ], + name: 'originating_host', + type: 'ip', + description: 'The host that will be initiating the data connection.\n', + }, + { + name: 'response_host', + type: 'ip', + description: 'The host that will be accepting the data connection.\n', + }, + { + name: 'response_port', + type: 'integer', + description: + 'The port at which the acceptor is listening for the data connection.\n', + }, + ], + }, + { + name: 'cwd', + type: 'keyword', + description: + "Current working directory that this session is in. By making the default value '.', we can indicate that unless something more concrete is discovered that the existing but unknown directory is ok to use.\n", + }, + { + name: 'cmdarg', + type: 'group', + description: 'Command that is currently waiting for a response.\n', + fields: [ + { + name: 'cmd', + type: 'keyword', + description: 'Command.\n', + }, + { + name: 'arg', + type: 'keyword', + description: 'Argument for the command if one was given.\n', + }, + { + name: 'seq', + type: 'integer', + description: 'Counter to track how many commands have been executed.\n', }, ], }, { - name: 'tls', + name: 'pending_commands', + type: 'integer', + description: + 'Queue for commands that have been sent but not yet responded to are tracked here.\n', + }, + { + name: 'passive', + type: 'boolean', + description: 'Indicates if the session is in active or passive mode.\n', + }, + { + name: 'capture_password', + type: 'boolean', + description: 'Determines if the password will be captured for this request.\n', + }, + { + name: 'last_auth_requested', + type: 'keyword', + description: + 'present if base/protocols/ftp/gridftp.bro is loaded.\nLast authentication/security mechanism that was used.\n', + }, + ], + }, + { + name: 'http', + type: 'group', + description: 'Fields exported by the Zeek HTTP log\n', + fields: [ + { + name: 'trans_depth', + type: 'integer', + description: + 'Represents the pipelined depth into the connection of this request/response transaction.\n', + }, + { + name: 'status_msg', + type: 'keyword', + description: 'Status message returned by the server.\n', + }, + { + name: 'info_code', + type: 'integer', + description: 'Last seen 1xx informational reply code returned by the server.\n', + }, + { + name: 'info_msg', + type: 'keyword', + description: 'Last seen 1xx informational reply message returned by the server.\n', + }, + { + name: 'tags', + type: 'keyword', + description: + 'A set of indicators of various attributes discovered and related to a particular\nrequest/response pair.\n', + }, + { + name: 'password', + type: 'keyword', + description: 'Password if basic-auth is performed for the request.\n', + }, + { + name: 'captured_password', + type: 'boolean', + description: 'Determines if the password will be captured for this request.\n', + }, + { + name: 'proxied', + type: 'keyword', + description: + 'All of the headers that may indicate if the HTTP request was proxied.\n', + }, + { + name: 'range_request', + type: 'boolean', + description: + 'Indicates if this request can assume 206 partial content in response.\n', + }, + { + name: 'client_header_names', + type: 'keyword', + description: + 'The vector of HTTP header names sent by the client. No header values\nare included here, just the header names.\n', + }, + { + name: 'server_header_names', + type: 'keyword', + description: + 'The vector of HTTP header names sent by the server. No header values\nare included here, just the header names.\n', + }, + { + name: 'orig_fuids', + type: 'keyword', + description: 'An ordered vector of file unique IDs from the originator.\n', + }, + { + name: 'orig_mime_types', + type: 'keyword', + description: 'An ordered vector of mime types from the originator.\n', + }, + { + name: 'orig_filenames', + type: 'keyword', + description: 'An ordered vector of filenames from the originator.\n', + }, + { + name: 'resp_fuids', + type: 'keyword', + description: 'An ordered vector of file unique IDs from the responder.\n', + }, + { + name: 'resp_mime_types', + type: 'keyword', + description: 'An ordered vector of mime types from the responder.\n', + }, + { + name: 'resp_filenames', + type: 'keyword', + description: 'An ordered vector of filenames from the responder.\n', + }, + { + name: 'orig_mime_depth', + type: 'integer', + description: 'Current number of MIME entities in the HTTP request message body.\n', + }, + { + name: 'resp_mime_depth', + type: 'integer', + description: 'Current number of MIME entities in the HTTP response message body.\n', + }, + ], + }, + { + name: 'intel', + type: 'group', + default_field: false, + description: 'Fields exported by the Zeek Intel log.\n', + fields: [ + { + name: 'seen', type: 'group', fields: [ { - name: 'notbefore', - type: 'date', + name: 'indicator', + type: 'keyword', + description: 'The intelligence indicator.\n', }, { - name: 'issuerdn', + name: 'indicator_type', type: 'keyword', + description: 'The type of data the indicator represents.\n', }, { - name: 'sni', + name: 'host', type: 'keyword', + description: + 'If the indicator type was Intel::ADDR, then this field will be present.\n', }, { - name: 'version', + name: 'conn', type: 'keyword', + description: + 'If the data was discovered within a connection, the connection record should go here to give context to the data.\n', }, { - name: 'session_resumed', - type: 'boolean', + name: 'where', + type: 'keyword', + description: 'Where the data was discovered.\n', }, { - name: 'fingerprint', + name: 'node', type: 'keyword', + description: 'The name of the node where the match was discovered.\n', }, { - name: 'serial', + name: 'uid', type: 'keyword', + description: + 'If the data was discovered within a connection, the connection uid should go here to give context to the data. If the conn field is provided, this will be automatically filled out.\n', }, { - name: 'notafter', - type: 'date', + name: 'f', + type: 'object', + description: + 'If the data was discovered within a file, the file record should go here to provide context to the data.\n', }, { - name: 'subject', + name: 'fuid', type: 'keyword', + description: + 'If the data was discovered within a file, the file uid should go here to provide context to the data. If the file record f is provided, this will be automatically filled out.\n', }, ], }, { - name: 'app_proto_ts', + name: 'matched', type: 'keyword', + description: + 'Event to represent a match in the intelligence data from data that was seen.\n', }, { - name: 'flow', + name: 'sources', + type: 'keyword', + description: 'Sources which supplied data for this match.\n', + }, + { + name: 'fuid', + type: 'keyword', + description: + 'If a file was associated with this intelligence hit, this is the uid for the file.\n', + }, + { + name: 'file_mime_type', + type: 'keyword', + description: + 'A mime type if the intelligence hit is related to a file. If the $f field is provided this will be automatically filled out.\n', + }, + { + name: 'file_desc', + type: 'keyword', + description: + 'Frequently files can be described to give a bit more context. If the $f field is provided this field will be automatically filled out.\n', + }, + ], + }, + { + name: 'irc', + type: 'group', + default_field: false, + description: 'Fields exported by the Zeek IRC log\n', + fields: [ + { + name: 'nick', + type: 'keyword', + description: 'Nickname given for the connection.\n', + }, + { + name: 'user', + type: 'keyword', + description: 'Username given for the connection.\n', + }, + { + name: 'command', + type: 'keyword', + description: 'Command given by the client.\n', + }, + { + name: 'value', + type: 'keyword', + description: 'Value for the command given by the client.\n', + }, + { + name: 'addl', + type: 'keyword', + description: 'Any additional data for the command.\n', + }, + { + name: 'dcc', type: 'group', fields: [ { - name: 'bytes_toclient', - type: 'alias', - path: 'destination.bytes', + name: 'file', + type: 'group', + fields: [ + { + name: 'name', + type: 'keyword', + description: + 'Present if base/protocols/irc/dcc-send.bro is loaded.\nDCC filename requested.\n', + }, + { + name: 'size', + type: 'long', + description: + 'Present if base/protocols/irc/dcc-send.bro is loaded.\nSize of the DCC transfer as indicated by the sender.\n', + }, + ], }, { - name: 'start', - type: 'alias', - path: 'event.start', + name: 'mime_type', + type: 'keyword', + description: + 'present if base/protocols/irc/dcc-send.bro is loaded.\nSniffed mime type of the file.\n', }, + ], + }, + { + name: 'fuid', + type: 'keyword', + description: + 'present if base/protocols/irc/files.bro is loaded.\nFile unique ID.\n', + }, + ], + }, + { + name: 'kerberos', + type: 'group', + default_field: false, + description: 'Fields exported by the Zeek Kerberos log\n', + fields: [ + { + name: 'request_type', + type: 'keyword', + description: + 'Request type - Authentication Service (AS) or Ticket Granting Service (TGS).\n', + }, + { + name: 'client', + type: 'keyword', + description: 'Client name.\n', + }, + { + name: 'service', + type: 'keyword', + description: 'Service name.\n', + }, + { + name: 'success', + type: 'boolean', + description: 'Request result.\n', + }, + { + name: 'error', + type: 'group', + fields: [ { - name: 'pkts_toclient', - type: 'alias', - path: 'destination.packets', + name: 'code', + type: 'integer', + description: 'Error code.\n', }, { - name: 'age', - type: 'long', + name: 'msg', + type: 'keyword', + description: 'Error message.\n', }, + ], + }, + { + name: 'valid', + type: 'group', + fields: [ { - name: 'state', - type: 'keyword', + name: 'from', + type: 'date', + description: 'Ticket valid from.\n', }, { - name: 'bytes_toserver', - type: 'alias', - path: 'source.bytes', + name: 'until', + type: 'date', + description: 'Ticket valid until.\n', }, { - name: 'reason', + name: 'days', + type: 'integer', + description: 'Number of days the ticket is valid for.\n', + }, + ], + }, + { + name: 'cipher', + type: 'keyword', + description: 'Ticket encryption type.\n', + }, + { + name: 'forwardable', + type: 'boolean', + description: 'Forwardable ticket requested.\n', + }, + { + name: 'renewable', + type: 'boolean', + description: 'Renewable ticket requested.\n', + }, + { + name: 'ticket', + type: 'group', + fields: [ + { + name: 'auth', type: 'keyword', + description: 'Hash of ticket used to authorize request/transaction.\n', }, { - name: 'pkts_toserver', - type: 'alias', - path: 'source.packets', + name: 'new', + type: 'keyword', + description: 'Hash of ticket returned by the KDC.\n', }, + ], + }, + { + name: 'cert', + type: 'group', + fields: [ { - name: 'end', - type: 'date', + name: 'client', + type: 'group', + fields: [ + { + name: 'value', + type: 'keyword', + description: 'Client certificate.\n', + }, + { + name: 'fuid', + type: 'keyword', + description: 'File unique ID of client cert.\n', + }, + { + name: 'subject', + type: 'keyword', + description: 'Subject of client certificate.\n', + }, + ], }, { - name: 'alerted', - type: 'boolean', + name: 'server', + type: 'group', + fields: [ + { + name: 'value', + type: 'keyword', + description: 'Server certificate.\n', + }, + { + name: 'fuid', + type: 'keyword', + description: 'File unique ID of server certificate.\n', + }, + { + name: 'subject', + type: 'keyword', + description: 'Subject of server certificate.\n', + }, + ], }, ], }, + ], + }, + { + name: 'modbus', + type: 'group', + default_field: false, + description: 'Fields exported by the Zeek modbus log.\n', + fields: [ { - name: 'app_proto', - type: 'alias', - path: 'network.protocol', + name: 'function', + type: 'keyword', + description: 'The name of the function message that was sent.\n', }, { - name: 'tx_id', - type: 'long', + name: 'exception', + type: 'keyword', + description: 'The exception if the response was a failure.\n', + }, + { + name: 'track_address', + type: 'integer', + description: + 'Present if policy/protocols/modbus/track-memmap.bro is loaded.\nModbus track address.\n', + }, + ], + }, + { + name: 'mysql', + type: 'group', + default_field: false, + description: 'Fields exported by the Zeek MySQL log.\n', + fields: [ + { + name: 'cmd', + type: 'keyword', + description: 'The command that was issued.\n', }, { - name: 'app_proto_tc', + name: 'arg', type: 'keyword', + description: 'The argument issued to the command.\n', }, { - name: 'smtp', - type: 'group', - fields: [ - { - name: 'rcpt_to', - type: 'keyword', - }, - { - name: 'mail_from', - type: 'keyword', - }, - { - name: 'helo', - type: 'keyword', - }, - ], + name: 'success', + type: 'boolean', + description: 'Whether the command succeeded.\n', }, { - name: 'app_proto_expected', - type: 'keyword', + name: 'rows', + type: 'integer', + description: 'The number of affected rows, if any.\n', }, { - name: 'flags', - type: 'group', + name: 'response', + type: 'keyword', + description: 'Server message, if any.\n', }, ], }, - ], - }, - ], - }, - { - key: 'zeek', - title: 'Zeek', - description: 'Module for handling logs produced by Zeek/Bro', - fields: [ - { - name: 'zeek', - type: 'group', - description: 'Fields from Zeek/Bro logs after normalization', - fields: [ - { - name: 'session_id', - type: 'keyword', - }, - { - name: 'connection.local_orig', - type: 'boolean', - }, - { - name: 'connection.local_resp', - type: 'boolean', - }, - { - name: 'connection.missed_bytes', - type: 'long', - }, - { - name: 'connection.state', - type: 'keyword', - }, - { - name: 'connection.history', - type: 'keyword', - }, - { - name: 'connection.orig_l2_addr', - type: 'keyword', - }, - { - name: 'resp_l2_addr', - type: 'keyword', - }, - { - name: 'vlan', - type: 'keyword', - }, - { - name: 'inner_vlan', - type: 'keyword', - }, - { - name: 'dns.trans_id', - type: 'integer', - }, - { - name: 'dns.rtt', - type: 'double', - }, - { - name: 'dns.query', - type: 'keyword', - }, - { - name: 'dns.qclass', - type: 'long', - }, - { - name: 'dns.qclass_name', - type: 'keyword', - }, - { - name: 'dns.qtype', - type: 'long', - }, - { - name: 'dns.qtype_name', - type: 'keyword', - }, - { - name: 'dns.rcode', - type: 'long', - }, - { - name: 'dns.rcode_name', - type: 'keyword', - }, - { - name: 'dns.AA', - type: 'boolean', - }, - { - name: 'dns.TC', - type: 'boolean', - }, - { - name: 'dns.RD', - type: 'boolean', - }, - { - name: 'dns.RA', - type: 'boolean', - }, - { - name: 'dns.answers', - type: 'keyword', - }, - { - name: 'dns.TTLs', - type: 'double', - }, - { - name: 'dns.rejected', - type: 'boolean', - }, - { - name: 'dns.total_answers', - type: 'integer', - }, - { - name: 'dns.total_replies', - type: 'integer', - }, - { - name: 'dns.saw_query', - type: 'boolean', - }, - { - name: 'dns.saw_reply', - type: 'boolean', - }, { - name: 'http.trans_depth', - type: 'integer', - }, - { - name: 'http.status_msg', - type: 'keyword', - }, - { - name: 'http.info_code', - type: 'integer', - }, - { - name: 'http.info_msg', - type: 'keyword', - }, - { - name: 'http.filename', - type: 'keyword', - }, - { - name: 'http.tags', - type: 'keyword', - }, - { - name: 'http.captured_password', - type: 'boolean', - }, - { - name: 'http.proxied', - type: 'keyword', - }, - { - name: 'http.range_request', - type: 'boolean', - }, - { - name: 'http.client_header_names', - type: 'keyword', - }, - { - name: 'http.server_header_names', - type: 'keyword', - }, - { - name: 'http.orig_fuids', - type: 'keyword', - }, - { - name: 'http.orig_mime_types', - type: 'keyword', - }, - { - name: 'http.orig_filenames', - type: 'keyword', - }, - { - name: 'http.resp_fuids', - type: 'keyword', - }, - { - name: 'http.resp_mime_types', - type: 'keyword', - }, - { - name: 'http.resp_filenames', - type: 'keyword', - }, - { - name: 'http.orig_mime_depth', - type: 'integer', - }, - { - name: 'http.resp_mime_depth', - type: 'integer', - }, - { - name: 'files.fuid', - type: 'keyword', - }, - { - name: 'files.tx_host', - type: 'ip', - }, - { - name: 'files.rx_host', - type: 'ip', - }, - { - name: 'files.session_ids', - type: 'keyword', - }, - { - name: 'files.source', - type: 'keyword', - }, - { - name: 'files.depth', - type: 'long', - }, - { - name: 'files.direction', - type: 'keyword', - }, - { - name: 'files.analyzers', - type: 'keyword', - }, - { - name: 'files.mime_type', - type: 'keyword', - }, - { - name: 'files.filename', - type: 'keyword', - }, - { - name: 'files.local_orig', - type: 'boolean', - }, - { - name: 'files.is_orig', - type: 'boolean', - }, - { - name: 'files.duration', - type: 'double', - }, - { - name: 'files.seen_bytes', - type: 'long', - }, - { - name: 'files.total_bytes', - type: 'long', - }, - { - name: 'files.missing_bytes', - type: 'long', - }, - { - name: 'files.overflow_bytes', - type: 'long', - }, - { - name: 'files.timedout', - type: 'boolean', - }, - { - name: 'files.parent_fuid', - type: 'keyword', - }, - { - name: 'files.md5', - type: 'keyword', - }, - { - name: 'files.sha1', - type: 'keyword', - }, - { - name: 'files.sha256', - type: 'keyword', - }, - { - name: 'files.extracted', - type: 'keyword', + name: 'notice', + type: 'group', + description: 'Fields exported by the Zeek Notice log.\n', + fields: [ + { + name: 'connection_id', + type: 'keyword', + description: 'Identifier of the related connection session.\n', + }, + { + name: 'icmp_id', + type: 'keyword', + description: 'Identifier of the related ICMP session.\n', + }, + { + name: 'file.id', + type: 'keyword', + description: + 'An identifier associated with a single file that is related to this notice.\n', + }, + { + name: 'file.parent_id', + type: 'keyword', + description: + 'Identifier associated with a container file from which this one was extracted.\n', + }, + { + name: 'file.source', + type: 'keyword', + description: + 'An identification of the source of the file data. E.g. it may be a network protocol\nover which it was transferred, or a local file path which was read, or some other\ninput source.\n', + }, + { + name: 'file.mime_type', + type: 'keyword', + description: 'A mime type if the notice is related to a file.\n', + }, + { + name: 'file.is_orig', + type: 'boolean', + description: + 'If the source of this file is a network connection, this field indicates if the file is\nbeing sent by the originator of the connection or the responder.\n', + }, + { + name: 'file.seen_bytes', + type: 'long', + description: 'Number of bytes provided to the file analysis engine for the file.\n', + }, + { + name: 'ffile.total_bytes', + type: 'long', + description: 'Total number of bytes that are supposed to comprise the full file.\n', + }, + { + name: 'file.missing_bytes', + type: 'long', + description: + 'The number of bytes in the file stream that were completely missed during the process\nof analysis.\n', + }, + { + name: 'file.overflow_bytes', + type: 'long', + description: + "The number of bytes in the file stream that were not delivered to stream file analyzers.\nThis could be overlapping bytes or bytes that couldn't be reassembled.\n", + }, + { + name: 'fuid', + type: 'keyword', + description: 'A file unique ID if this notice is related to a file.\n', + }, + { + name: 'note', + type: 'keyword', + description: 'The type of the notice.\n', + }, + { + name: 'msg', + type: 'keyword', + description: 'The human readable message for the notice.\n', + }, + { + name: 'sub', + type: 'keyword', + description: 'The human readable sub-message.\n', + }, + { + name: 'n', + type: 'long', + description: 'Associated count, or a status code.\n', + }, + { + name: 'peer_name', + type: 'keyword', + description: 'Name of remote peer that raised this notice.\n', + }, + { + name: 'peer_descr', + type: 'text', + description: 'Textual description for the peer that raised this notice.\n', + }, + { + name: 'actions', + type: 'keyword', + description: 'The actions which have been applied to this notice.\n', + }, + { + name: 'email_body_sections', + type: 'text', + description: + 'By adding chunks of text into this element, other scripts can expand on notices\nthat are being emailed.\n', + }, + { + name: 'email_delay_tokens', + type: 'keyword', + description: + 'Adding a string token to this set will cause the built-in emailing functionality\nto delay sending the email either the token has been removed or the email\nhas been delayed for the specified time duration.\n', + }, + { + name: 'identifier', + type: 'keyword', + description: + 'This field is provided when a notice is generated for the purpose of deduplicating notices.\n', + }, + { + name: 'suppress_for', + type: 'double', + description: + 'This field indicates the length of time that this unique notice should be suppressed.\n', + }, + { + name: 'dropped', + type: 'boolean', + description: + 'Indicate if the source IP address was dropped and denied network access.\n', + }, + ], }, { - name: 'files.extracted_cutoff', - type: 'boolean', + name: 'ntlm', + type: 'group', + default_field: false, + description: 'Fields exported by the Zeek NTLM log.\n', + fields: [ + { + name: 'domain', + type: 'keyword', + description: 'Domain name given by the client.\n', + }, + { + name: 'hostname', + type: 'keyword', + description: 'Hostname given by the client.\n', + }, + { + name: 'success', + type: 'boolean', + description: 'Indicate whether or not the authentication was successful.\n', + }, + { + name: 'username', + type: 'keyword', + description: 'Username given by the client.\n', + }, + { + name: 'server', + type: 'group', + fields: [ + { + name: 'name', + type: 'group', + fields: [ + { + name: 'dns', + type: 'keyword', + description: 'DNS name given by the server in a CHALLENGE.\n', + }, + { + name: 'netbios', + type: 'keyword', + description: 'NetBIOS name given by the server in a CHALLENGE.\n', + }, + { + name: 'tree', + type: 'keyword', + description: 'Tree name given by the server in a CHALLENGE.\n', + }, + ], + }, + ], + }, + ], }, { - name: 'files.extracted_size', - type: 'long', + name: 'ocsp', + type: 'group', + default_field: false, + description: + 'Fields exported by the Zeek OCSP log\nOnline Certificate Status Protocol (OCSP). Only created if policy script is loaded.\n', + fields: [ + { + name: 'file_id', + type: 'keyword', + description: 'File id of the OCSP reply.\n', + }, + { + name: 'hash', + type: 'group', + fields: [ + { + name: 'algorithm', + type: 'keyword', + description: + 'Hash algorithm used to generate issuerNameHash and issuerKeyHash.\n', + }, + { + name: 'issuer', + type: 'group', + fields: [ + { + name: 'name', + type: 'keyword', + description: "Hash of the issuer's distingueshed name.\n", + }, + { + name: 'key', + type: 'keyword', + description: "Hash of the issuer's public key.\n", + }, + ], + }, + ], + }, + { + name: 'serial_number', + type: 'keyword', + description: 'Serial number of the affected certificate.\n', + }, + { + name: 'status', + type: 'keyword', + description: 'Status of the affected certificate.\n', + }, + { + name: 'revoke', + type: 'group', + fields: [ + { + name: 'time', + type: 'date', + description: 'Time at which the certificate was revoked.\n', + }, + { + name: 'reason', + type: 'keyword', + description: 'Reason for which the certificate was revoked.\n', + }, + ], + }, + { + name: 'update', + type: 'group', + fields: [ + { + name: 'this', + type: 'date', + description: + 'The time at which the status being shows is known to have been correct.\n', + }, + { + name: 'next', + type: 'date', + description: + 'The latest time at which new information about the status of the certificate will be available.\n', + }, + ], + }, + ], }, { - name: 'files.entropy', - type: 'double', + name: 'pe', + type: 'group', + default_field: false, + description: 'Fields exported by the Zeek pe log.\n', + fields: [ + { + name: 'client', + type: 'keyword', + description: "The client's version string.\n", + }, + { + name: 'id', + type: 'keyword', + description: 'File id of this portable executable file.\n', + }, + { + name: 'machine', + type: 'keyword', + description: 'The target machine that the file was compiled for.\n', + }, + { + name: 'compile_time', + type: 'date', + description: 'The time that the file was created at.\n', + }, + { + name: 'os', + type: 'keyword', + description: 'The required operating system.\n', + }, + { + name: 'subsystem', + type: 'keyword', + description: 'The subsystem that is required to run this file.\n', + }, + { + name: 'is_exe', + type: 'boolean', + description: 'Is the file an executable, or just an object file?\n', + }, + { + name: 'is_64bit', + type: 'boolean', + description: 'Is the file a 64-bit executable?\n', + }, + { + name: 'uses_aslr', + type: 'boolean', + description: 'Does the file support Address Space Layout Randomization?\n', + }, + { + name: 'uses_dep', + type: 'boolean', + description: 'Does the file support Data Execution Prevention?\n', + }, + { + name: 'uses_code_integrity', + type: 'boolean', + description: 'Does the file enforce code integrity checks?\n', + }, + { + name: 'uses_seh', + type: 'boolean', + description: 'Does the file use structured exception handing?\n', + }, + { + name: 'has_import_table', + type: 'boolean', + description: 'Does the file have an import table?\n', + }, + { + name: 'has_export_table', + type: 'boolean', + description: 'Does the file have an export table?\n', + }, + { + name: 'has_cert_table', + type: 'boolean', + description: 'Does the file have an attribute certificate table?\n', + }, + { + name: 'has_debug_data', + type: 'boolean', + description: 'Does the file have a debug table?\n', + }, + { + name: 'section_names', + type: 'keyword', + description: 'The names of the sections, in order.\n', + }, + ], }, { - name: 'ssl.version', - type: 'keyword', + name: 'radius', + type: 'group', + default_field: false, + description: 'Fields exported by the Zeek Radius log.\n', + fields: [ + { + name: 'username', + type: 'keyword', + description: 'The username, if present.\n', + }, + { + name: 'mac', + type: 'keyword', + description: 'MAC address, if present.\n', + }, + { + name: 'framed_addr', + type: 'ip', + description: + 'The address given to the network access server, if present. This is only a hint from the RADIUS server and the network access server is not required to honor the address.\n', + }, + { + name: 'remote_ip', + type: 'ip', + description: + 'Remote IP address, if present. This is collected from the Tunnel-Client-Endpoint attribute.\n', + }, + { + name: 'connect_info', + type: 'keyword', + description: 'Connect info, if present.\n', + }, + { + name: 'reply_msg', + type: 'keyword', + description: + 'Reply message from the server challenge. This is frequently shown to the user authenticating.\n', + }, + { + name: 'result', + type: 'keyword', + description: 'Successful or failed authentication.\n', + }, + { + name: 'ttl', + type: 'integer', + description: + 'The duration between the first request and either the "Access-Accept" message or an error. If the field is empty, it means that either the request or response was not seen.\n', + }, + { + name: 'logged', + type: 'boolean', + description: 'Whether this has already been logged and can be ignored.\n', + }, + ], }, { - name: 'ssl.cipher', - type: 'keyword', + name: 'rdp', + type: 'group', + default_field: false, + description: 'Fields exported by the Zeek RDP log.\n', + fields: [ + { + name: 'cookie', + type: 'keyword', + description: + 'Cookie value used by the client machine. This is typically a username.\n', + }, + { + name: 'result', + type: 'keyword', + description: + "Status result for the connection. It's a mix between RDP negotation failure messages and GCC server create response messages.\n", + }, + { + name: 'security_protocol', + type: 'keyword', + description: 'Security protocol chosen by the server.\n', + }, + { + name: 'keyboard_layout', + type: 'keyword', + description: 'Keyboard layout (language) of the client machine.\n', + }, + { + name: 'client', + type: 'group', + fields: [ + { + name: 'build', + type: 'keyword', + description: 'RDP client version used by the client machine.\n', + }, + { + name: 'client_name', + type: 'keyword', + description: 'Name of the client machine.\n', + }, + { + name: 'product_id', + type: 'keyword', + description: 'Product ID of the client machine.\n', + }, + ], + }, + { + name: 'desktop', + type: 'group', + fields: [ + { + name: 'width', + type: 'integer', + description: 'Desktop width of the client machine.\n', + }, + { + name: 'height', + type: 'integer', + description: 'Desktop height of the client machine.\n', + }, + { + name: 'color_depth', + type: 'keyword', + description: + 'The color depth requested by the client in the high_color_depth field.\n', + }, + ], + }, + { + name: 'cert', + type: 'group', + fields: [ + { + name: 'type', + type: 'keyword', + description: + 'If the connection is being encrypted with native RDP encryption, this is the type of cert being used.\n', + }, + { + name: 'count', + type: 'integer', + description: + 'The number of certs seen. X.509 can transfer an entire certificate chain.\n', + }, + { + name: 'permanent', + type: 'boolean', + description: + 'Indicates if the provided certificate or certificate chain is permanent or temporary.\n', + }, + ], + }, + { + name: 'encryption', + type: 'group', + fields: [ + { + name: 'level', + type: 'keyword', + description: 'Encryption level of the connection.\n', + }, + { + name: 'method', + type: 'keyword', + description: 'Encryption method of the connection.\n', + }, + ], + }, + { + name: 'done', + type: 'boolean', + description: 'Track status of logging RDP connections.\n', + }, + { + name: 'ssl', + type: 'boolean', + description: + '(present if policy/protocols/rdp/indicate_ssl.bro is loaded)\nFlag the connection if it was seen over SSL.\n', + }, + ], }, { - name: 'ssl.curve', - type: 'keyword', + name: 'rfb', + type: 'group', + default_field: false, + description: 'Fields exported by the Zeek RFB log.\n', + fields: [ + { + name: 'version', + type: 'group', + fields: [ + { + name: 'client', + type: 'group', + fields: [ + { + name: 'major', + type: 'keyword', + description: 'Major version of the client.\n', + }, + { + name: 'minor', + type: 'keyword', + description: 'Minor version of the client.\n', + }, + ], + }, + { + name: 'server', + type: 'group', + fields: [ + { + name: 'major', + type: 'keyword', + description: 'Major version of the server.\n', + }, + { + name: 'minor', + type: 'keyword', + description: 'Minor version of the server.\n', + }, + ], + }, + ], + }, + { + name: 'auth', + type: 'group', + fields: [ + { + name: 'success', + type: 'boolean', + description: 'Whether or not authentication was successful.\n', + }, + { + name: 'method', + type: 'keyword', + description: 'Identifier of authentication method used.\n', + }, + ], + }, + { + name: 'share_flag', + type: 'boolean', + description: 'Whether the client has an exclusive or a shared session.\n', + }, + { + name: 'desktop_name', + type: 'keyword', + description: 'Name of the screen that is being shared.\n', + }, + { + name: 'width', + type: 'integer', + description: 'Width of the screen that is being shared.\n', + }, + { + name: 'height', + type: 'integer', + description: 'Height of the screen that is being shared.\n', + }, + ], }, { - name: 'ssl.server_name', - type: 'keyword', + name: 'sip', + type: 'group', + default_field: false, + description: 'Fields exported by the Zeek SIP log.\n', + fields: [ + { + name: 'transaction_depth', + type: 'integer', + description: + 'Represents the pipelined depth into the connection of this request/response transaction.\n', + }, + { + name: 'sequence', + type: 'group', + fields: [ + { + name: 'method', + type: 'keyword', + description: 'Verb used in the SIP request (INVITE, REGISTER etc.).\n', + }, + { + name: 'number', + type: 'keyword', + description: 'Contents of the CSeq: header from the client.\n', + }, + ], + }, + { + name: 'uri', + type: 'keyword', + description: 'URI used in the request.\n', + }, + { + name: 'date', + type: 'keyword', + description: 'Contents of the Date: header from the client.\n', + }, + { + name: 'request', + type: 'group', + fields: [ + { + name: 'from', + type: 'keyword', + description: + "Contents of the request From: header Note: The tag= value that's usually appended to the sender is stripped off and not logged.\n", + }, + { + name: 'to', + type: 'keyword', + description: 'Contents of the To: header.\n', + }, + { + name: 'path', + type: 'keyword', + description: + 'The client message transmission path, as extracted from the headers.\n', + }, + { + name: 'body_length', + type: 'long', + description: 'Contents of the Content-Length: header from the client.\n', + }, + ], + }, + { + name: 'response', + type: 'group', + fields: [ + { + name: 'from', + type: 'keyword', + description: + "Contents of the response From: header Note: The tag= value that's usually appended to the sender is stripped off and not logged.\n", + }, + { + name: 'to', + type: 'keyword', + description: 'Contents of the response To: header.\n', + }, + { + name: 'path', + type: 'keyword', + description: + 'The server message transmission path, as extracted from the headers.\n', + }, + { + name: 'body_length', + type: 'long', + description: 'Contents of the Content-Length: header from the server.\n', + }, + ], + }, + { + name: 'reply_to', + type: 'keyword', + description: 'Contents of the Reply-To: header.\n', + }, + { + name: 'call_id', + type: 'keyword', + description: 'Contents of the Call-ID: header from the client.\n', + }, + { + name: 'subject', + type: 'keyword', + description: 'Contents of the Subject: header from the client.\n', + }, + { + name: 'user_agent', + type: 'keyword', + description: 'Contents of the User-Agent: header from the client.\n', + }, + { + name: 'status', + type: 'group', + fields: [ + { + name: 'code', + type: 'integer', + description: 'Status code returned by the server.\n', + }, + { + name: 'msg', + type: 'keyword', + description: 'Status message returned by the server.\n', + }, + ], + }, + { + name: 'warning', + type: 'keyword', + description: 'Contents of the Warning: header.\n', + }, + { + name: 'content_type', + type: 'keyword', + description: 'Contents of the Content-Type: header from the server.\n', + }, + ], }, { - name: 'ssl.resumed', - type: 'boolean', + name: 'smb_cmd', + type: 'group', + default_field: false, + description: 'Fields exported by the Zeek smb_cmd log.\n', + fields: [ + { + name: 'command', + type: 'keyword', + description: 'The command sent by the client.\n', + }, + { + name: 'sub_command', + type: 'keyword', + description: 'The subcommand sent by the client, if present.\n', + }, + { + name: 'argument', + type: 'keyword', + description: 'Command argument sent by the client, if any.\n', + }, + { + name: 'status', + type: 'keyword', + description: "Server reply to the client's command.\n", + }, + { + name: 'rtt', + type: 'double', + description: 'Round trip time from the request to the response.\n', + }, + { + name: 'version', + type: 'keyword', + description: 'Version of SMB for the command.\n', + }, + { + name: 'username', + type: 'keyword', + description: 'Authenticated username, if available.\n', + }, + { + name: 'tree', + type: 'keyword', + description: + 'If this is related to a tree, this is the tree that was used for the current command.\n', + }, + { + name: 'tree_service', + type: 'keyword', + description: 'The type of tree (disk share, printer share, named pipe, etc.).\n', + }, + { + name: 'file', + type: 'group', + description: 'If the command referenced a file, store it here.\n', + fields: [ + { + name: 'name', + type: 'keyword', + description: 'Filename if one was seen.\n', + }, + { + name: 'action', + type: 'keyword', + description: 'Action this log record represents.\n', + }, + { + name: 'uid', + type: 'keyword', + description: 'UID of the referenced file.\n', + }, + { + name: 'host', + type: 'group', + fields: [ + { + name: 'tx', + type: 'ip', + description: 'Address of the transmitting host.\n', + }, + { + name: 'rx', + type: 'ip', + description: 'Address of the receiving host.\n', + }, + ], + }, + ], + }, + { + name: 'smb1_offered_dialects', + type: 'keyword', + description: + 'Present if base/protocols/smb/smb1-main.bro is loaded.\nDialects offered by the client.\n', + }, + { + name: 'smb2_offered_dialects', + type: 'integer', + description: + 'Present if base/protocols/smb/smb2-main.bro is loaded.\nDialects offered by the client.\n', + }, + ], }, { - name: 'ssl.next_protocol', - type: 'keyword', + name: 'smb_files', + type: 'group', + default_field: false, + description: 'Fields exported by the Zeek SMB Files log.\n', + fields: [ + { + name: 'action', + type: 'keyword', + description: 'Action this log record represents.\n', + }, + { + name: 'fid', + type: 'integer', + description: 'ID referencing this file.\n', + }, + { + name: 'name', + type: 'keyword', + description: 'Filename if one was seen.\n', + }, + { + name: 'path', + type: 'keyword', + description: 'Path pulled from the tree this file was transferred to or from.\n', + }, + { + name: 'previous_name', + type: 'keyword', + description: + "If the rename action was seen, this will be the file's previous name.\n", + }, + { + name: 'size', + type: 'long', + description: 'Byte size of the file.\n', + }, + { + name: 'times', + type: 'group', + description: 'Timestamps of the file.\n', + fields: [ + { + name: 'accessed', + type: 'date', + description: "The file's access time.\n", + }, + { + name: 'changed', + type: 'date', + description: "The file's change time.\n", + }, + { + name: 'created', + type: 'date', + description: "The file's create time.\n", + }, + { + name: 'modified', + type: 'date', + description: "The file's modify time.\n", + }, + ], + }, + { + name: 'uuid', + type: 'keyword', + description: 'UUID referencing this file if DCE/RPC.\n', + }, + ], }, { - name: 'ssl.established', - type: 'boolean', + name: 'smb_mapping', + type: 'group', + default_field: false, + description: 'Fields exported by the Zeek SMB_Mapping log.\n', + fields: [ + { + name: 'path', + type: 'keyword', + description: 'Name of the tree path.\n', + }, + { + name: 'service', + type: 'keyword', + description: + 'The type of resource of the tree (disk share, printer share, named pipe, etc.).\n', + }, + { + name: 'native_file_system', + type: 'keyword', + description: 'File system of the tree.\n', + }, + { + name: 'share_type', + type: 'keyword', + description: + 'If this is SMB2, a share type will be included. For SMB1, the type of share\nwill be deduced and included as well.\n', + }, + ], }, { - name: 'ssl.cert_chain', - type: 'keyword', + name: 'smtp', + type: 'group', + default_field: false, + description: 'Fields exported by the Zeek SMTP log.\n', + fields: [ + { + name: 'transaction_depth', + type: 'integer', + description: + 'A count to represent the depth of this message transaction in a single connection where multiple messages were transferred.\n', + }, + { + name: 'helo', + type: 'keyword', + description: 'Contents of the Helo header.\n', + }, + { + name: 'mail_from', + type: 'keyword', + description: 'Email addresses found in the MAIL FROM header.\n', + }, + { + name: 'rcpt_to', + type: 'keyword', + description: 'Email addresses found in the RCPT TO header.\n', + }, + { + name: 'date', + type: 'date', + description: 'Contents of the Date header.\n', + }, + { + name: 'from', + type: 'keyword', + description: 'Contents of the From header.\n', + }, + { + name: 'to', + type: 'keyword', + description: 'Contents of the To header.\n', + }, + { + name: 'cc', + type: 'keyword', + description: 'Contents of the CC header.\n', + }, + { + name: 'reply_to', + type: 'keyword', + description: 'Contents of the ReplyTo header.\n', + }, + { + name: 'msg_id', + type: 'keyword', + description: 'Contents of the MsgID header.\n', + }, + { + name: 'in_reply_to', + type: 'keyword', + description: 'Contents of the In-Reply-To header.\n', + }, + { + name: 'subject', + type: 'keyword', + description: 'Contents of the Subject header.\n', + }, + { + name: 'x_originating_ip', + type: 'keyword', + description: 'Contents of the X-Originating-IP header.\n', + }, + { + name: 'first_received', + type: 'keyword', + description: 'Contents of the first Received header.\n', + }, + { + name: 'second_received', + type: 'keyword', + description: 'Contents of the second Received header.\n', + }, + { + name: 'last_reply', + type: 'keyword', + description: 'The last message that the server sent to the client.\n', + }, + { + name: 'path', + type: 'ip', + description: 'The message transmission path, as extracted from the headers.\n', + }, + { + name: 'user_agent', + type: 'keyword', + description: 'Value of the User-Agent header from the client.\n', + }, + { + name: 'tls', + type: 'boolean', + description: 'Indicates that the connection has switched to using TLS.\n', + }, + { + name: 'process_received_from', + type: 'boolean', + description: + 'Indicates if the "Received: from" headers should still be processed.\n', + }, + { + name: 'has_client_activity', + type: 'boolean', + description: 'Indicates if client activity has been seen, but not yet logged.\n', + }, + { + name: 'fuids', + type: 'keyword', + description: + '(present if base/protocols/smtp/files.bro is loaded)\nAn ordered vector of file unique IDs seen attached to the message.\n', + }, + { + name: 'is_webmail', + type: 'boolean', + description: 'Indicates if the message was sent through a webmail interface.\n', + }, + ], }, { - name: 'ssl.cert_chain_fuids', - type: 'keyword', + name: 'snmp', + type: 'group', + default_field: false, + description: 'Fields exported by the Zeek SNMP log.\n', + fields: [ + { + name: 'duration', + type: 'double', + description: + 'The amount of time between the first packet beloning to the SNMP session and the latest one seen.\n', + }, + { + name: 'version', + type: 'keyword', + description: 'The version of SNMP being used.\n', + }, + { + name: 'community', + type: 'keyword', + description: + "The community string of the first SNMP packet associated with the session. This is used as part of SNMP's (v1 and v2c) administrative/security framework. See RFC 1157 or RFC 1901.\n", + }, + { + name: 'get', + type: 'group', + fields: [ + { + name: 'requests', + type: 'integer', + description: + 'The number of variable bindings in GetRequest/GetNextRequest PDUs seen for the session.\n', + }, + { + name: 'bulk_requests', + type: 'integer', + description: + 'The number of variable bindings in GetBulkRequest PDUs seen for the session.\n', + }, + { + name: 'responses', + type: 'integer', + description: + 'The number of variable bindings in GetResponse/Response PDUs seen for the session.\n', + }, + ], + }, + { + name: 'set', + type: 'group', + fields: [ + { + name: 'requests', + type: 'integer', + description: + 'The number of variable bindings in SetRequest PDUs seen for the session.\n', + }, + ], + }, + { + name: 'display_string', + type: 'keyword', + description: 'A system description of the SNMP responder endpoint.\n', + }, + { + name: 'up_since', + type: 'date', + description: + "The time at which the SNMP responder endpoint claims it's been up since.\n", + }, + ], }, { - name: 'ssl.client_cert_chain', - type: 'keyword', + name: 'socks', + type: 'group', + default_field: false, + description: 'Fields exported by the Zeek SOCKS log.\n', + fields: [ + { + name: 'version', + type: 'integer', + description: 'Protocol version of SOCKS.\n', + }, + { + name: 'user', + type: 'keyword', + description: 'Username used to request a login to the proxy.\n', + }, + { + name: 'password', + type: 'keyword', + description: 'Password used to request a login to the proxy.\n', + }, + { + name: 'status', + type: 'keyword', + description: 'Server status for the attempt at using the proxy.\n', + }, + { + name: 'request', + type: 'group', + fields: [ + { + name: 'host', + type: 'keyword', + description: + 'Client requested SOCKS address. Could be an address, a name or both.\n', + }, + { + name: 'port', + type: 'integer', + description: 'Client requested port.\n', + }, + ], + }, + { + name: 'bound', + type: 'group', + fields: [ + { + name: 'host', + type: 'keyword', + description: 'Server bound address. Could be an address, a name or both.\n', + }, + { + name: 'port', + type: 'integer', + description: 'Server bound port.\n', + }, + ], + }, + { + name: 'capture_password', + type: 'boolean', + description: 'Determines if the password will be captured for this request.\n', + }, + ], }, { - name: 'ssl.client_cert_chain_fuids', - type: 'keyword', + name: 'ssh', + type: 'group', + default_field: false, + description: 'Fields exported by the Zeek SSH log.\n', + fields: [ + { + name: 'client', + type: 'keyword', + description: "The client's version string.\n", + }, + { + name: 'direction', + type: 'keyword', + description: + 'Direction of the connection. If the client was a local host logging into\nan external host, this would be OUTBOUND. INBOUND would be set for the\nopposite situation.\n', + }, + { + name: 'host_key', + type: 'keyword', + description: "The server's key thumbprint.\n", + }, + { + name: 'server', + type: 'keyword', + description: "The server's version string.\n", + }, + { + name: 'version', + type: 'integer', + description: 'SSH major version (1 or 2).\n', + }, + { + name: 'algorithm', + type: 'group', + description: 'Cipher algorithms used in this session.\n', + fields: [ + { + name: 'cipher', + type: 'keyword', + description: 'The encryption algorithm in use.\n', + }, + { + name: 'compression', + type: 'keyword', + description: 'The compression algorithm in use.\n', + }, + { + name: 'host_key', + type: 'keyword', + description: "The server host key's algorithm.\n", + }, + { + name: 'key_exchange', + type: 'keyword', + description: 'The key exchange algorithm in use.\n', + }, + { + name: 'mac', + type: 'keyword', + description: 'The signing (MAC) algorithm in use.\n', + }, + ], + }, + { + name: 'auth', + type: 'group', + fields: [ + { + name: 'attempts', + type: 'integer', + description: + "The number of authentication attemps we observed. There's always at\nleast one, since some servers might support no authentication at all.\nIt's important to note that not all of these are failures, since some\nservers require two-factor auth (e.g. password AND pubkey).\n", + }, + { + name: 'success', + type: 'boolean', + description: 'Authentication result.\n', + }, + ], + }, + ], }, { - name: 'ssl.issuer', - type: 'keyword', + name: 'ssl', + type: 'group', + default_field: false, + description: 'Fields exported by the Zeek SSL log.\n', + fields: [ + { + name: 'version', + type: 'keyword', + description: 'SSL/TLS version that was logged.\n', + }, + { + name: 'cipher', + type: 'keyword', + description: 'SSL/TLS cipher suite that was logged.\n', + }, + { + name: 'curve', + type: 'keyword', + description: 'Elliptic curve that was logged when using ECDH/ECDHE.\n', + }, + { + name: 'resumed', + type: 'boolean', + description: + 'Flag to indicate if the session was resumed reusing the key material exchanged in an\nearlier connection.\n', + }, + { + name: 'next_protocol', + type: 'keyword', + description: + 'Next protocol the server chose using the application layer next protocol extension.\n', + }, + { + name: 'established', + type: 'boolean', + description: + 'Flag to indicate if this ssl session has been established successfully.\n', + }, + { + name: 'validation', + type: 'group', + fields: [ + { + name: 'status', + type: 'keyword', + description: 'Result of certificate validation for this connection.\n', + }, + { + name: 'code', + type: 'keyword', + description: + 'Result of certificate validation for this connection, given as OpenSSL validation code.\n', + }, + ], + }, + { + name: 'last_alert', + type: 'keyword', + description: 'Last alert that was seen during the connection.\n', + }, + { + name: 'server', + type: 'group', + fields: [ + { + name: 'name', + type: 'keyword', + description: + 'Value of the Server Name Indicator SSL/TLS extension. It indicates the server name\nthat the client was requesting.\n', + }, + { + name: 'cert_chain', + type: 'keyword', + description: + 'Chain of certificates offered by the server to validate its complete signing chain.\n', + }, + { + name: 'cert_chain_fuids', + type: 'keyword', + description: + 'An ordered vector of certificate file identifiers for the certificates offered by the server.\n', + }, + { + name: 'issuer', + type: 'group', + description: + 'Subject of the signer of the X.509 certificate offered by the server.\n', + fields: [ + { + name: 'common_name', + type: 'keyword', + description: + 'Common name of the signer of the X.509 certificate offered by the server.\n', + }, + { + name: 'country', + type: 'keyword', + description: + 'Country code of the signer of the X.509 certificate offered by the server.\n', + }, + { + name: 'locality', + type: 'keyword', + description: + 'Locality of the signer of the X.509 certificate offered by the server.\n', + }, + { + name: 'organization', + type: 'keyword', + description: + 'Organization of the signer of the X.509 certificate offered by the server.\n', + }, + { + name: 'organizational_unit', + type: 'keyword', + description: + 'Organizational unit of the signer of the X.509 certificate offered by the server.\n', + }, + { + name: 'state', + type: 'keyword', + description: + 'State or province name of the signer of the X.509 certificate offered by the server.\n', + }, + ], + }, + { + name: 'subject', + type: 'group', + description: 'Subject of the X.509 certificate offered by the server.\n', + fields: [ + { + name: 'common_name', + type: 'keyword', + description: + 'Common name of the X.509 certificate offered by the server.\n', + }, + { + name: 'country', + type: 'keyword', + description: + 'Country code of the X.509 certificate offered by the server.\n', + }, + { + name: 'locality', + type: 'keyword', + description: 'Locality of the X.509 certificate offered by the server.\n', + }, + { + name: 'organization', + type: 'keyword', + description: + 'Organization of the X.509 certificate offered by the server.\n', + }, + { + name: 'organizational_unit', + type: 'keyword', + description: + 'Organizational unit of the X.509 certificate offered by the server.\n', + }, + { + name: 'state', + type: 'keyword', + description: + 'State or province name of the X.509 certificate offered by the server.\n', + }, + ], + }, + ], + }, + { + name: 'client', + type: 'group', + fields: [ + { + name: 'cert_chain', + type: 'keyword', + description: + 'Chain of certificates offered by the client to validate its complete signing chain.\n', + }, + { + name: 'cert_chain_fuids', + type: 'keyword', + description: + 'An ordered vector of certificate file identifiers for the certificates offered by the client.\n', + }, + { + name: 'issuer', + type: 'group', + description: + 'Subject of the signer of the X.509 certificate offered by the client.\n', + fields: [ + { + name: 'common_name', + type: 'keyword', + description: + 'Common name of the signer of the X.509 certificate offered by the client.\n', + }, + { + name: 'country', + type: 'keyword', + description: + 'Country code of the signer of the X.509 certificate offered by the client.\n', + }, + { + name: 'locality', + type: 'keyword', + description: + 'Locality of the signer of the X.509 certificate offered by the client.\n', + }, + { + name: 'organization', + type: 'keyword', + description: + 'Organization of the signer of the X.509 certificate offered by the client.\n', + }, + { + name: 'organizational_unit', + type: 'keyword', + description: + 'Organizational unit of the signer of the X.509 certificate offered by the client.\n', + }, + { + name: 'state', + type: 'keyword', + description: + 'State or province name of the signer of the X.509 certificate offered by the client.\n', + }, + ], + }, + { + name: 'subject', + type: 'group', + description: 'Subject of the X.509 certificate offered by the client.\n', + fields: [ + { + name: 'common_name', + type: 'keyword', + description: + 'Common name of the X.509 certificate offered by the client.\n', + }, + { + name: 'country', + type: 'keyword', + description: + 'Country code of the X.509 certificate offered by the client.\n', + }, + { + name: 'locality', + type: 'keyword', + description: 'Locality of the X.509 certificate offered by the client.\n', + }, + { + name: 'organization', + type: 'keyword', + description: + 'Organization of the X.509 certificate offered by the client.\n', + }, + { + name: 'organizational_unit', + type: 'keyword', + description: + 'Organizational unit of the X.509 certificate offered by the client.\n', + }, + { + name: 'state', + type: 'keyword', + description: + 'State or province name of the X.509 certificate offered by the client.\n', + }, + ], + }, + ], + }, + ], }, { - name: 'ssl.client_issuer', - type: 'keyword', + name: 'stats', + type: 'group', + default_field: false, + description: 'Fields exported by the Zeek stats log.\n', + fields: [ + { + name: 'peer', + type: 'keyword', + description: 'Peer that generated this log. Mostly for clusters.\n', + }, + { + name: 'memory', + type: 'integer', + description: 'Amount of memory currently in use in MB.\n', + }, + { + name: 'packets', + type: 'group', + fields: [ + { + name: 'processed', + type: 'long', + description: 'Number of packets processed since the last stats interval.\n', + }, + { + name: 'dropped', + type: 'long', + description: + 'Number of packets dropped since the last stats interval if reading live traffic.\n', + }, + { + name: 'received', + type: 'long', + description: + 'Number of packets seen on the link since the last stats interval if reading live traffic.\n', + }, + ], + }, + { + name: 'bytes', + type: 'group', + fields: [ + { + name: 'received', + type: 'long', + description: + 'Number of bytes received since the last stats interval if reading live traffic.\n', + }, + ], + }, + { + name: 'connections', + type: 'group', + fields: [ + { + name: 'tcp', + type: 'group', + fields: [ + { + name: 'active', + type: 'integer', + description: 'TCP connections currently in memory.\n', + }, + { + name: 'count', + type: 'integer', + description: 'TCP connections seen since last stats interval.\n', + }, + ], + }, + { + name: 'udp', + type: 'group', + fields: [ + { + name: 'active', + type: 'integer', + description: 'UDP connections currently in memory.\n', + }, + { + name: 'count', + type: 'integer', + description: 'UDP connections seen since last stats interval.\n', + }, + ], + }, + { + name: 'icmp', + type: 'group', + fields: [ + { + name: 'active', + type: 'integer', + description: 'ICMP connections currently in memory.\n', + }, + { + name: 'count', + type: 'integer', + description: 'ICMP connections seen since last stats interval.\n', + }, + ], + }, + ], + }, + { + name: 'events', + type: 'group', + fields: [ + { + name: 'processed', + type: 'integer', + description: 'Number of events processed since the last stats interval.\n', + }, + { + name: 'queued', + type: 'integer', + description: + 'Number of events that have been queued since the last stats interval.\n', + }, + ], + }, + { + name: 'timers', + type: 'group', + fields: [ + { + name: 'count', + type: 'integer', + description: 'Number of timers scheduled since last stats interval.\n', + }, + { + name: 'active', + type: 'integer', + description: 'Current number of scheduled timers.\n', + }, + ], + }, + { + name: 'files', + type: 'group', + fields: [ + { + name: 'count', + type: 'integer', + description: 'Number of files seen since last stats interval.\n', + }, + { + name: 'active', + type: 'integer', + description: 'Current number of files actively being seen.\n', + }, + ], + }, + { + name: 'dns_requests', + type: 'group', + fields: [ + { + name: 'count', + type: 'integer', + description: 'Number of DNS requests seen since last stats interval.\n', + }, + { + name: 'active', + type: 'integer', + description: 'Current number of DNS requests awaiting a reply.\n', + }, + ], + }, + { + name: 'reassembly_size', + type: 'group', + fields: [ + { + name: 'tcp', + type: 'integer', + description: 'Current size of TCP data in reassembly.\n', + }, + { + name: 'file', + type: 'integer', + description: 'Current size of File data in reassembly.\n', + }, + { + name: 'frag', + type: 'integer', + description: 'Current size of packet fragment data in reassembly.\n', + }, + { + name: 'unknown', + type: 'integer', + description: + 'Current size of unknown data in reassembly (this is only PIA buffer right now).\n', + }, + ], + }, + { + name: 'timestamp_lag', + type: 'integer', + description: + 'Lag between the wall clock and packet timestamps if reading live traffic.\n', + }, + ], }, { - name: 'ssl.validation_status', - type: 'keyword', + name: 'syslog', + type: 'group', + default_field: false, + description: 'Fields exported by the Zeek syslog log.\n', + fields: [ + { + name: 'facility', + type: 'keyword', + description: 'Syslog facility for the message.\n', + }, + { + name: 'severity', + type: 'keyword', + description: 'Syslog severity for the message.\n', + }, + { + name: 'message', + type: 'keyword', + description: 'The plain text message.\n', + }, + ], }, { - name: 'ssl.subject', - type: 'keyword', + name: 'tunnel', + type: 'group', + default_field: false, + description: 'Fields exported by the Zeek SSH log.\n', + fields: [ + { + name: 'type', + type: 'keyword', + description: 'The type of tunnel.\n', + }, + { + name: 'action', + type: 'keyword', + description: 'The type of activity that occurred.\n', + }, + ], }, { - name: 'ssl.client_subject', - type: 'keyword', + name: 'weird', + type: 'group', + default_field: false, + description: 'Fields exported by the Zeek Weird log.\n', + fields: [ + { + name: 'name', + type: 'keyword', + description: 'The name of the weird that occurred.\n', + }, + { + name: 'additional_info', + type: 'keyword', + description: 'Additional information accompanying the weird if any.\n', + }, + { + name: 'notice', + type: 'boolean', + description: 'Indicate if this weird was also turned into a notice.\n', + }, + { + name: 'peer', + type: 'keyword', + description: + 'The peer that originated this weird. This is helpful in cluster deployments if a particular cluster node is having trouble to help identify which node is having trouble.\n', + }, + { + name: 'identifier', + type: 'keyword', + description: + 'This field is to be provided when a weird is generated for the purpose of deduplicating weirds. The identifier string should be unique for a single instance of the weird. This field is used to define when a weird is conceptually a duplicate of a previous weird.\n', + }, + ], }, { - name: 'ssl.last_alert', - type: 'keyword', + name: 'x509', + type: 'group', + default_field: false, + description: 'Fields exported by the Zeek x509 log.\n', + fields: [ + { + name: 'id', + type: 'keyword', + description: 'File id of this certificate.\n', + }, + { + name: 'certificate', + type: 'group', + description: 'Basic information about the certificate.\n', + fields: [ + { + name: 'version', + type: 'integer', + description: 'Version number.\n', + }, + { + name: 'serial', + type: 'keyword', + description: 'Serial number.\n', + }, + { + name: 'subject', + type: 'group', + description: 'Subject.\n', + fields: [ + { + name: 'country', + type: 'keyword', + description: 'Country provided in the certificate subject.\n', + }, + { + name: 'common_name', + type: 'keyword', + description: 'Common name provided in the certificate subject.\n', + }, + { + name: 'locality', + type: 'keyword', + description: 'Locality provided in the certificate subject.\n', + }, + { + name: 'organization', + type: 'keyword', + description: 'Organization provided in the certificate subject.\n', + }, + { + name: 'organizational_unit', + type: 'keyword', + description: 'Organizational unit provided in the certificate subject.\n', + }, + { + name: 'state', + type: 'keyword', + description: 'State or province provided in the certificate subject.\n', + }, + ], + }, + { + name: 'issuer', + type: 'group', + description: 'Issuer.\n', + fields: [ + { + name: 'country', + type: 'keyword', + description: 'Country provided in the certificate issuer field.\n', + }, + { + name: 'common_name', + type: 'keyword', + description: 'Common name provided in the certificate issuer field.\n', + }, + { + name: 'locality', + type: 'keyword', + description: 'Locality provided in the certificate issuer field.\n', + }, + { + name: 'organization', + type: 'keyword', + description: 'Organization provided in the certificate issuer field.\n', + }, + { + name: 'organizational_unit', + type: 'keyword', + description: + 'Organizational unit provided in the certificate issuer field.\n', + }, + { + name: 'state', + type: 'keyword', + description: + 'State or province provided in the certificate issuer field.\n', + }, + ], + }, + { + name: 'common_name', + type: 'keyword', + description: 'Last (most specific) common name.\n', + }, + { + name: 'valid', + type: 'group', + description: 'Certificate validity timestamps\n', + fields: [ + { + name: 'from', + type: 'date', + description: 'Timestamp before when certificate is not valid.\n', + }, + { + name: 'until', + type: 'date', + description: 'Timestamp after when certificate is not valid.\n', + }, + ], + }, + { + name: 'key', + type: 'group', + fields: [ + { + name: 'algorithm', + type: 'keyword', + description: 'Name of the key algorithm.\n', + }, + { + name: 'type', + type: 'keyword', + description: + 'Key type, if key parseable by openssl (either rsa, dsa or ec).\n', + }, + { + name: 'length', + type: 'integer', + description: 'Key length in bits.\n', + }, + ], + }, + { + name: 'signature_algorithm', + type: 'keyword', + description: 'Name of the signature algorithm.\n', + }, + { + name: 'exponent', + type: 'keyword', + description: 'Exponent, if RSA-certificate.\n', + }, + { + name: 'curve', + type: 'keyword', + description: 'Curve, if EC-certificate.\n', + }, + ], + }, + { + name: 'san', + type: 'group', + description: 'Subject alternative name extension of the certificate.\n', + fields: [ + { + name: 'dns', + type: 'keyword', + description: 'List of DNS entries in SAN.\n', + }, + { + name: 'uri', + type: 'keyword', + description: 'List of URI entries in SAN.\n', + }, + { + name: 'email', + type: 'keyword', + description: 'List of email entries in SAN.\n', + }, + { + name: 'ip', + type: 'ip', + description: 'List of IP entries in SAN.\n', + }, + { + name: 'other_fields', + type: 'boolean', + description: + 'True if the certificate contained other, not recognized or parsed name fields.\n', + }, + ], + }, + { + name: 'basic_constraints', + type: 'group', + description: 'Basic constraints extension of the certificate.\n', + fields: [ + { + name: 'certificate_authority', + type: 'boolean', + description: 'CA flag set or not.\n', + }, + { + name: 'path_length', + type: 'integer', + description: 'Maximum path length.\n', + }, + ], + }, + { + name: 'log_cert', + type: 'boolean', + description: + 'Present if policy/protocols/ssl/log-hostcerts-only.bro is loaded\nLogging of certificate is suppressed if set to F.\n', + }, + ], }, ], }, @@ -7293,47 +18360,47 @@ export const filebeatSchema: Schema = [ { key: 'netflow', title: 'NetFlow', - description: 'Fields from NetFlow and IPFIX flows.', + description: 'Fields from NetFlow and IPFIX flows.\n', fields: [ { name: 'netflow', type: 'group', - description: 'Fields from NetFlow and IPFIX.', + description: 'Fields from NetFlow and IPFIX.\n', fields: [ { name: 'type', type: 'keyword', - description: 'The type of NetFlow record described by this event.', + description: 'The type of NetFlow record described by this event.\n', }, { name: 'exporter', type: 'group', - description: 'Metadata related to the exporter device that generated this record.', + description: 'Metadata related to the exporter device that generated this record.\n', fields: [ { name: 'address', type: 'keyword', - description: "Exporter's network address in IP:port format. ", + description: "Exporter's network address in IP:port format.\n", }, { name: 'source_id', type: 'long', - description: 'Observation domain ID to which this record belongs.', + description: 'Observation domain ID to which this record belongs.\n', }, { name: 'timestamp', type: 'date', - description: 'Time and date of export.', + description: 'Time and date of export.\n', }, { name: 'uptime_millis', type: 'long', - description: 'How long the exporter process has been running, in milliseconds.', + description: 'How long the exporter process has been running, in milliseconds.\n', }, { name: 'version', - type: 'long', - description: 'NetFlow version used.', + type: 'integer', + description: 'NetFlow version used.\n', }, ], }, @@ -7539,7 +18606,7 @@ export const filebeatSchema: Schema = [ }, { name: 'class_id', - type: 'short', + type: 'long', }, { name: 'minimum_ttl', @@ -8118,19 +19185,19 @@ export const filebeatSchema: Schema = [ type: 'long', }, { - name: 'post_nast_ource_ipv4_address', + name: 'post_nat_source_ipv4_address', type: 'ip', }, { - name: 'post_nadt_estination_ipv4_address', + name: 'post_nat_destination_ipv4_address', type: 'ip', }, { - name: 'post_napst_ource_transport_port', + name: 'post_napt_source_transport_port', type: 'integer', }, { - name: 'post_napdt_estination_transport_port', + name: 'post_napt_destination_transport_port', type: 'integer', }, { @@ -8342,11 +19409,11 @@ export const filebeatSchema: Schema = [ type: 'long', }, { - name: 'post_nast_ource_ipv6_address', + name: 'post_nat_source_ipv6_address', type: 'ip', }, { - name: 'post_nadt_estination_ipv6_address', + name: 'post_nat_destination_ipv6_address', type: 'ip', }, { @@ -8514,11 +19581,11 @@ export const filebeatSchema: Schema = [ type: 'long', }, { - name: 'hash_ipp_ayload_offset', + name: 'hash_ip_payload_offset', type: 'long', }, { - name: 'hash_ipp_ayload_size', + name: 'hash_ip_payload_size', type: 'long', }, { @@ -8550,11 +19617,11 @@ export const filebeatSchema: Schema = [ type: 'keyword', }, { - name: 'upper_cli_imit', + name: 'upper_ci_limit', type: 'double', }, { - name: 'lower_cli_imit', + name: 'lower_ci_limit', type: 'double', }, { @@ -8718,11 +19785,11 @@ export const filebeatSchema: Schema = [ type: 'long', }, { - name: 'distinct_count_of_sourc_eipa_ddress', + name: 'distinct_count_of_source_ip_address', type: 'long', }, { - name: 'distinct_count_of_destinatio_nipa_ddress', + name: 'distinct_count_of_destination_ip_address', type: 'long', }, { @@ -8782,11 +19849,11 @@ export const filebeatSchema: Schema = [ type: 'long', }, { - name: 'selector_itd_otal_flows_observed', + name: 'selector_id_total_flows_observed', type: 'long', }, { - name: 'selector_itd_otal_flows_selected', + name: 'selector_id_total_flows_selected', type: 'long', }, { @@ -8950,7 +20017,7 @@ export const filebeatSchema: Schema = [ type: 'short', }, { - name: 'mib_object_valuei_pa_ddress', + name: 'mib_object_value_ip_address', type: 'ip', }, { @@ -9078,7 +20145,7 @@ export const filebeatSchema: Schema = [ type: 'long', }, { - name: 'max_bieb_ntries', + name: 'max_bib_entries', type: 'long', }, { @@ -9125,4 +20192,1052 @@ export const filebeatSchema: Schema = [ }, ], }, + { + key: 's3', + title: 's3', + description: 'S3 fields from s3 input.\n', + release: 'beta', + fields: [ + { + name: 'bucket_name', + type: 'keyword', + description: 'Name of the S3 bucket that this log retrieved from.\n', + }, + { + name: 'object_key', + type: 'keyword', + description: 'Name of the S3 object that this log retrieved from.\n', + }, + ], + }, + { + key: 'cef', + title: 'Decode CEF processor fields', + description: 'Common Event Format (CEF) data.\n', + fields: [ + { + name: 'cef', + type: 'group', + description: + 'By default the `decode_cef` processor writes all data from the CEF message to this `cef` object. It contains the CEF header fields and the extension data.\n', + fields: [ + { + name: 'version', + type: 'keyword', + description: 'Version of the CEF specification used by the message.\n', + }, + { + name: 'device.vendor', + type: 'keyword', + description: 'Vendor of the device that produced the message.\n', + }, + { + name: 'device.product', + type: 'keyword', + description: 'Product of the device that produced the message.\n', + }, + { + name: 'device.version', + type: 'keyword', + description: 'Version of the product that produced the message.\n', + }, + { + name: 'device.event_class_id', + type: 'keyword', + description: 'Unique identifier of the event type.\n', + }, + { + name: 'severity', + type: 'keyword', + example: 'Very-High', + description: + 'Importance of the event. The valid string values are Unknown, Low, Medium, High, and Very-High. The valid integer values are 0-3=Low, 4-6=Medium, 7- 8=High, and 9-10=Very-High.\n', + }, + { + name: 'name', + type: 'keyword', + description: 'Short description of the event.\n', + }, + { + name: 'extensions', + type: 'group', + description: 'Collection of key-value pairs carried in the CEF extension field.\n', + default_field: false, + fields: [ + { + name: 'agentAddress', + type: 'ip', + description: 'The IP address of the ArcSight connector that processed the event.', + }, + { + name: 'agentDnsDomain', + type: 'keyword', + description: + 'The DNS domain name of the ArcSight connector that processed the event.', + }, + { + name: 'agentHostName', + type: 'keyword', + description: 'The hostname of the ArcSight connector that processed the event.', + }, + { + name: 'agentId', + type: 'keyword', + description: 'The agent ID of the ArcSight connector that processed the event.', + }, + { + name: 'agentMacAddress', + type: 'keyword', + description: 'The MAC address of the ArcSight connector that processed the event.', + }, + { + name: 'agentNtDomain', + type: 'keyword', + description: '', + }, + { + name: 'agentReceiptTime', + type: 'date', + description: + 'The time at which information about the event was received by the ArcSight connector.', + }, + { + name: 'agentTimeZone', + type: 'keyword', + description: + 'The agent time zone of the ArcSight connector that processed the event.', + }, + { + name: 'agentTranslatedAddress', + type: 'ip', + description: '', + }, + { + name: 'agentTranslatedZoneExternalID', + type: 'keyword', + description: '', + }, + { + name: 'agentTranslatedZoneURI', + type: 'keyword', + description: '', + }, + { + name: 'agentType', + type: 'keyword', + description: 'The agent type of the ArcSight connector that processed the event', + }, + { + name: 'agentVersion', + type: 'keyword', + description: 'The version of the ArcSight connector that processed the event.', + }, + { + name: 'agentZoneExternalID', + type: 'keyword', + description: '', + }, + { + name: 'agentZoneURI', + type: 'keyword', + description: '', + }, + { + name: 'applicationProtocol', + type: 'keyword', + description: + 'Application level protocol, example values are HTTP, HTTPS, SSHv2, Telnet, POP, IMPA, IMAPS, and so on.', + }, + { + name: 'baseEventCount', + type: 'long', + description: + 'A count associated with this event. How many times was this same event observed? Count can be omitted if it is 1.', + }, + { + name: 'bytesIn', + type: 'long', + description: + 'Number of bytes transferred inbound, relative to the source to destination relationship, meaning that data was flowing from source to destination.', + }, + { + name: 'bytesOut', + type: 'long', + description: + 'Number of bytes transferred outbound relative to the source to destination relationship. For example, the byte number of data flowing from the destination to the source.', + }, + { + name: 'customerExternalID', + type: 'keyword', + description: '', + }, + { + name: 'customerURI', + type: 'keyword', + description: '', + }, + { + name: 'destinationAddress', + type: 'ip', + description: + 'Identifies the destination address that the event refers to in an IP network. The format is an IPv4 address.', + }, + { + name: 'destinationDnsDomain', + type: 'keyword', + description: + 'The DNS domain part of the complete fully qualified domain name (FQDN).', + }, + { + name: 'destinationGeoLatitude', + type: 'double', + description: + "The latitudinal value from which the destination's IP address belongs.", + }, + { + name: 'destinationGeoLongitude', + type: 'double', + description: + "The longitudinal value from which the destination's IP address belongs.", + }, + { + name: 'destinationHostName', + type: 'keyword', + description: + 'Identifies the destination that an event refers to in an IP network. The format should be a fully qualified domain name (FQDN) associated with the destination node, when a node is available.', + }, + { + name: 'destinationMacAddress', + type: 'keyword', + description: 'Six colon-seperated hexadecimal numbers.', + }, + { + name: 'destinationNtDomain', + type: 'keyword', + description: 'The Windows domain name of the destination address.', + }, + { + name: 'destinationPort', + type: 'long', + description: 'The valid port numbers are between 0 and 65535.', + }, + { + name: 'destinationProcessId', + type: 'long', + description: + 'Provides the ID of the destination process associated with the event. For example, if an event contains process ID 105, "105" is the process ID.', + }, + { + name: 'destinationProcessName', + type: 'keyword', + description: "The name of the event's destination process.", + }, + { + name: 'destinationServiceName', + type: 'keyword', + description: 'The service targeted by this event.', + }, + { + name: 'destinationTranslatedAddress', + type: 'ip', + description: + 'Identifies the translated destination that the event refers to in an IP network.', + }, + { + name: 'destinationTranslatedPort', + type: 'long', + description: + 'Port after it was translated; for example, a firewall. Valid port numbers are 0 to 65535.', + }, + { + name: 'destinationTranslatedZoneExternalID', + type: 'keyword', + description: '', + }, + { + name: 'destinationTranslatedZoneURI', + type: 'keyword', + description: + 'The URI for the Translated Zone that the destination asset has been assigned to in ArcSight.', + }, + { + name: 'destinationUserId', + type: 'keyword', + description: + 'Identifies the destination user by ID. For example, in UNIX, the root user is generally associated with user ID 0.', + }, + { + name: 'destinationUserName', + type: 'keyword', + description: + "Identifies the destination user by name. This is the user associated with the event's destination. Email addresses are often mapped into the UserName fields. The recipient is a candidate to put into this field.", + }, + { + name: 'destinationUserPrivileges', + type: 'keyword', + description: + 'The typical values are "Administrator", "User", and "Guest". This identifies the destination user\'s privileges. In UNIX, for example, activity executed on the root user would be identified with destinationUser Privileges of "Administrator".', + }, + { + name: 'destinationZoneExternalID', + type: 'keyword', + description: '', + }, + { + name: 'destinationZoneURI', + type: 'keyword', + description: + 'The URI for the Zone that the destination asset has been assigned to in ArcSight.', + }, + { + name: 'deviceAction', + type: 'keyword', + description: 'Action taken by the device.', + }, + { + name: 'deviceAddress', + type: 'ip', + description: + 'Identifies the device address that an event refers to in an IP network.', + }, + { + name: 'deviceCustomFloatingPoint1Label', + type: 'keyword', + description: + 'All custom fields have a corresponding label field. Each of these fields is a string and describes the purpose of the custom field.', + }, + { + name: 'deviceCustomFloatingPoint3Label', + type: 'keyword', + description: + 'All custom fields have a corresponding label field. Each of these fields is a string and describes the purpose of the custom field.', + }, + { + name: 'deviceCustomFloatingPoint4Label', + type: 'keyword', + description: + 'All custom fields have a corresponding label field. Each of these fields is a string and describes the purpose of the custom field.', + }, + { + name: 'deviceCustomDate1', + type: 'date', + description: + 'One of two timestamp fields available to map fields that do not apply to any other in this dictionary.', + }, + { + name: 'deviceCustomDate1Label', + type: 'keyword', + description: + 'All custom fields have a corresponding label field. Each of these fields is a string and describes the purpose of the custom field.', + }, + { + name: 'deviceCustomDate2', + type: 'date', + description: + 'One of two timestamp fields available to map fields that do not apply to any other in this dictionary.', + }, + { + name: 'deviceCustomDate2Label', + type: 'keyword', + description: + 'All custom fields have a corresponding label field. Each of these fields is a string and describes the purpose of the custom field.', + }, + { + name: 'deviceCustomFloatingPoint1', + type: 'double', + description: + 'One of four floating point fields available to map fields that do not apply to any other in this dictionary.', + }, + { + name: 'deviceCustomFloatingPoint2', + type: 'double', + description: + 'One of four floating point fields available to map fields that do not apply to any other in this dictionary.', + }, + { + name: 'deviceCustomFloatingPoint2Label', + type: 'keyword', + description: + 'All custom fields have a corresponding label field. Each of these fields is a string and describes the purpose of the custom field.', + }, + { + name: 'deviceCustomFloatingPoint3', + type: 'double', + description: + 'One of four floating point fields available to map fields that do not apply to any other in this dictionary.', + }, + { + name: 'deviceCustomFloatingPoint4', + type: 'double', + description: + 'One of four floating point fields available to map fields that do not apply to any other in this dictionary.', + }, + { + name: 'deviceCustomIPv6Address1', + type: 'ip', + description: + 'One of four IPv6 address fields available to map fields that do not apply to any other in this dictionary.', + }, + { + name: 'deviceCustomIPv6Address1Label', + type: 'keyword', + description: + 'All custom fields have a corresponding label field. Each of these fields is a string and describes the purpose of the custom field.', + }, + { + name: 'deviceCustomIPv6Address2', + type: 'ip', + description: + 'One of four IPv6 address fields available to map fields that do not apply to any other in this dictionary.', + }, + { + name: 'deviceCustomIPv6Address2Label', + type: 'keyword', + description: + 'All custom fields have a corresponding label field. Each of these fields is a string and describes the purpose of the custom field.', + }, + { + name: 'deviceCustomIPv6Address3', + type: 'ip', + description: + 'One of four IPv6 address fields available to map fields that do not apply to any other in this dictionary.', + }, + { + name: 'deviceCustomIPv6Address3Label', + type: 'keyword', + description: + 'All custom fields have a corresponding label field. Each of these fields is a string and describes the purpose of the custom field.', + }, + { + name: 'deviceCustomIPv6Address4', + type: 'ip', + description: + 'One of four IPv6 address fields available to map fields that do not apply to any other in this dictionary.', + }, + { + name: 'deviceCustomIPv6Address4Label', + type: 'keyword', + description: + 'All custom fields have a corresponding label field. Each of these fields is a string and describes the purpose of the custom field.', + }, + { + name: 'deviceCustomNumber1', + type: 'long', + description: + 'One of three number fields available to map fields that do not apply to any other in this dictionary. Use sparingly and seek a more specific, dictionary supplied field when possible.', + }, + { + name: 'deviceCustomNumber1Label', + type: 'keyword', + description: + 'All custom fields have a corresponding label field. Each of these fields is a string and describes the purpose of the custom field.', + }, + { + name: 'deviceCustomNumber2', + type: 'long', + description: + 'One of three number fields available to map fields that do not apply to any other in this dictionary. Use sparingly and seek a more specific, dictionary supplied field when possible.', + }, + { + name: 'deviceCustomNumber2Label', + type: 'keyword', + description: + 'All custom fields have a corresponding label field. Each of these fields is a string and describes the purpose of the custom field.', + }, + { + name: 'deviceCustomNumber3', + type: 'long', + description: + 'One of three number fields available to map fields that do not apply to any other in this dictionary. Use sparingly and seek a more specific, dictionary supplied field when possible.', + }, + { + name: 'deviceCustomNumber3Label', + type: 'keyword', + description: + 'All custom fields have a corresponding label field. Each of these fields is a string and describes the purpose of the custom field.', + }, + { + name: 'deviceCustomString1', + type: 'keyword', + description: + 'One of six strings available to map fields that do not apply to any other in this dictionary. Use sparingly and seek a more specific, dictionary supplied field when possible.', + }, + { + name: 'deviceCustomString1Label', + type: 'keyword', + description: + 'All custom fields have a corresponding label field. Each of these fields is a string and describes the purpose of the custom field.', + }, + { + name: 'deviceCustomString2', + type: 'keyword', + description: + 'One of six strings available to map fields that do not apply to any other in this dictionary. Use sparingly and seek a more specific, dictionary supplied field when possible.', + }, + { + name: 'deviceCustomString2Label', + type: 'keyword', + description: + 'All custom fields have a corresponding label field. Each of these fields is a string and describes the purpose of the custom field.', + }, + { + name: 'deviceCustomString3', + type: 'keyword', + description: + 'One of six strings available to map fields that do not apply to any other in this dictionary. Use sparingly and seek a more specific, dictionary supplied field when possible.', + }, + { + name: 'deviceCustomString3Label', + type: 'keyword', + description: + 'All custom fields have a corresponding label field. Each of these fields is a string and describes the purpose of the custom field.', + }, + { + name: 'deviceCustomString4', + type: 'keyword', + description: + 'One of six strings available to map fields that do not apply to any other in this dictionary. Use sparingly and seek a more specific, dictionary supplied field when possible.', + }, + { + name: 'deviceCustomString4Label', + type: 'keyword', + description: + 'All custom fields have a corresponding label field. Each of these fields is a string and describes the purpose of the custom field.', + }, + { + name: 'deviceCustomString5', + type: 'keyword', + description: + 'One of six strings available to map fields that do not apply to any other in this dictionary. Use sparingly and seek a more specific, dictionary supplied field when possible.', + }, + { + name: 'deviceCustomString5Label', + type: 'keyword', + description: + 'All custom fields have a corresponding label field. Each of these fields is a string and describes the purpose of the custom field.', + }, + { + name: 'deviceCustomString6', + type: 'keyword', + description: + 'One of six strings available to map fields that do not apply to any other in this dictionary. Use sparingly and seek a more specific, dictionary supplied field when possible.', + }, + { + name: 'deviceCustomString6Label', + type: 'keyword', + description: + 'All custom fields have a corresponding label field. Each of these fields is a string and describes the purpose of the custom field.', + }, + { + name: 'deviceDirection', + type: 'long', + description: + 'Any information about what direction the observed communication has taken. The following values are supported - "0" for inbound or "1" for outbound.', + }, + { + name: 'deviceDnsDomain', + type: 'keyword', + description: + 'The DNS domain part of the complete fully qualified domain name (FQDN).', + }, + { + name: 'deviceEventCategory', + type: 'keyword', + description: + 'Represents the category assigned by the originating device. Devices often use their own categorization schema to classify event. Example "/Monitor/Disk/Read".', + }, + { + name: 'deviceExternalId', + type: 'keyword', + description: 'A name that uniquely identifies the device generating this event.', + }, + { + name: 'deviceFacility', + type: 'keyword', + description: + 'The facility generating this event. For example, Syslog has an explicit facility associated with every event.', + }, + { + name: 'deviceFlexNumber1', + type: 'long', + description: + 'One of two alternative number fields available to map fields that do not apply to any other in this dictionary. Use sparingly and seek a more specific, dictionary supplied field when possible.', + }, + { + name: 'deviceFlexNumber1Label', + type: 'keyword', + description: + 'All custom fields have a corresponding label field. Each of these fields is a string and describes the purpose of the custom field.', + }, + { + name: 'deviceFlexNumber2', + type: 'long', + description: + 'One of two alternative number fields available to map fields that do not apply to any other in this dictionary. Use sparingly and seek a more specific, dictionary supplied field when possible.', + }, + { + name: 'deviceFlexNumber2Label', + type: 'keyword', + description: + 'All custom fields have a corresponding label field. Each of these fields is a string and describes the purpose of the custom field.', + }, + { + name: 'deviceHostName', + type: 'keyword', + description: + 'The format should be a fully qualified domain name (FQDN) associated with the device node, when a node is available.', + }, + { + name: 'deviceInboundInterface', + type: 'keyword', + description: 'Interface on which the packet or data entered the device.', + }, + { + name: 'deviceMacAddress', + type: 'keyword', + description: 'Six colon-separated hexadecimal numbers.', + }, + { + name: 'deviceNtDomain', + type: 'keyword', + description: 'The Windows domain name of the device address.', + }, + { + name: 'deviceOutboundInterface', + type: 'keyword', + description: 'Interface on which the packet or data left the device.', + }, + { + name: 'devicePayloadId', + type: 'keyword', + description: 'Unique identifier for the payload associated with the event.', + }, + { + name: 'deviceProcessId', + type: 'long', + description: 'Provides the ID of the process on the device generating the event.', + }, + { + name: 'deviceProcessName', + type: 'keyword', + description: + 'Process name associated with the event. An example might be the process generating the syslog entry in UNIX.', + }, + { + name: 'deviceReceiptTime', + type: 'date', + description: + 'The time at which the event related to the activity was received. The format is MMM dd yyyy HH:mm:ss or milliseconds since epoch (Jan 1st 1970)', + }, + { + name: 'deviceTimeZone', + type: 'keyword', + description: 'The timezone for the device generating the event.', + }, + { + name: 'deviceTranslatedAddress', + type: 'ip', + description: + 'Identifies the translated device address that the event refers to in an IP network.', + }, + { + name: 'deviceTranslatedZoneExternalID', + type: 'keyword', + description: '', + }, + { + name: 'deviceTranslatedZoneURI', + type: 'keyword', + description: + 'The URI for the Translated Zone that the device asset has been assigned to in ArcSight.', + }, + { + name: 'deviceZoneExternalID', + type: 'keyword', + description: '', + }, + { + name: 'deviceZoneURI', + type: 'keyword', + description: + 'Thee URI for the Zone that the device asset has been assigned to in ArcSight.', + }, + { + name: 'endTime', + type: 'date', + description: + 'The time at which the activity related to the event ended. The format is MMM dd yyyy HH:mm:ss or milliseconds since epoch (Jan 1st1970). An example would be reporting the end of a session.', + }, + { + name: 'eventId', + type: 'long', + description: 'This is a unique ID that ArcSight assigns to each event.', + }, + { + name: 'eventOutcome', + type: 'keyword', + description: "Displays the outcome, usually as 'success' or 'failure'.", + }, + { + name: 'externalId', + type: 'keyword', + description: + 'The ID used by an originating device. They are usually increasing numbers, associated with events.', + }, + { + name: 'fileCreateTime', + type: 'date', + description: 'Time when the file was created.', + }, + { + name: 'fileHash', + type: 'keyword', + description: 'Hash of a file.', + }, + { + name: 'fileId', + type: 'keyword', + description: 'An ID associated with a file could be the inode.', + }, + { + name: 'fileModificationTime', + type: 'date', + description: 'Time when the file was last modified.', + }, + { + name: 'filename', + type: 'keyword', + description: 'Name of the file only (without its path).', + }, + { + name: 'filePath', + type: 'keyword', + description: 'Full path to the file, including file name itself.', + }, + { + name: 'filePermission', + type: 'keyword', + description: 'Permissions of the file.', + }, + { + name: 'fileSize', + type: 'long', + description: 'Size of the file.', + }, + { + name: 'fileType', + type: 'keyword', + description: 'Type of file (pipe, socket, etc.)', + }, + { + name: 'flexDate1', + type: 'date', + description: + 'A timestamp field available to map a timestamp that does not apply to any other defined timestamp field in this dictionary. Use all flex fields sparingly and seek a more specific, dictionary supplied field when possible. These fields are typically reserved for customer use and should not be set by vendors unless necessary.', + }, + { + name: 'flexDate1Label', + type: 'keyword', + description: + 'The label field is a string and describes the purpose of the flex field.', + }, + { + name: 'flexString1', + type: 'keyword', + description: + 'One of four floating point fields available to map fields that do not apply to any other in this dictionary. Use sparingly and seek a more specific, dictionary supplied field when possible. These fields are typically reserved for customer use and should not be set by vendors unless necessary.', + }, + { + name: 'flexString2', + type: 'keyword', + description: + 'One of four floating point fields available to map fields that do not apply to any other in this dictionary. Use sparingly and seek a more specific, dictionary supplied field when possible. These fields are typically reserved for customer use and should not be set by vendors unless necessary.', + }, + { + name: 'flexString1Label', + type: 'keyword', + description: + 'The label field is a string and describes the purpose of the flex field.', + }, + { + name: 'flexString2Label', + type: 'keyword', + description: + 'The label field is a string and describes the purpose of the flex field.', + }, + { + name: 'message', + type: 'keyword', + description: + 'An arbitrary message giving more details about the event. Multi-line entries can be produced by using \\n as the new line separator.', + }, + { + name: 'oldFileCreateTime', + type: 'date', + description: 'Time when old file was created.', + }, + { + name: 'oldFileHash', + type: 'keyword', + description: 'Hash of the old file.', + }, + { + name: 'oldFileId', + type: 'keyword', + description: 'An ID associated with the old file could be the inode.', + }, + { + name: 'oldFileModificationTime', + type: 'date', + description: 'Time when old file was last modified.', + }, + { + name: 'oldFileName', + type: 'keyword', + description: 'Name of the old file.', + }, + { + name: 'oldFilePath', + type: 'keyword', + description: 'Full path to the old file, including the file name itself.', + }, + { + name: 'oldFilePermission', + type: 'keyword', + description: 'Permissions of the old file.', + }, + { + name: 'oldFileSize', + type: 'long', + description: 'Size of the old file.', + }, + { + name: 'oldFileType', + type: 'keyword', + description: 'Type of the old file (pipe, socket, etc.)', + }, + { + name: 'rawEvent', + type: 'keyword', + description: '', + }, + { + name: 'Reason', + type: 'keyword', + description: + 'The reason an audit event was generated. For example "bad password" or "unknown user". This could also be an error or return code. Example "0x1234".', + }, + { + name: 'requestClientApplication', + type: 'keyword', + description: 'The User-Agent associated with the request.', + }, + { + name: 'requestContext', + type: 'keyword', + description: + 'Description of the content from which the request originated (for example, HTTP Referrer)', + }, + { + name: 'requestCookies', + type: 'keyword', + description: 'Cookies associated with the request.', + }, + { + name: 'requestMethod', + type: 'keyword', + description: 'The HTTP method used to access a URL.', + }, + { + name: 'requestUrl', + type: 'keyword', + description: + 'In the case of an HTTP request, this field contains the URL accessed. The URL should contain the protocol as well.', + }, + { + name: 'sourceAddress', + type: 'ip', + description: 'Identifies the source that an event refers to in an IP network.', + }, + { + name: 'sourceDnsDomain', + type: 'keyword', + description: + 'The DNS domain part of the complete fully qualified domain name (FQDN).', + }, + { + name: 'sourceGeoLatitude', + type: 'double', + description: '', + }, + { + name: 'sourceGeoLongitude', + type: 'double', + description: '', + }, + { + name: 'sourceHostName', + type: 'keyword', + description: + "Identifies the source that an event refers to in an IP network. The format should be a fully qualified domain name (FQDN) associated with the source node, when a mode is available. Examples: 'host' or 'host.domain.com'.\n", + }, + { + name: 'sourceMacAddress', + type: 'keyword', + example: '00:0d:60:af:1b:61', + description: 'Six colon-separated hexadecimal numbers.', + }, + { + name: 'sourceNtDomain', + type: 'keyword', + description: 'The Windows domain name for the source address.', + }, + { + name: 'sourcePort', + type: 'long', + description: 'The valid port numbers are 0 to 65535.', + }, + { + name: 'sourceProcessId', + type: 'long', + description: 'The ID of the source process associated with the event.', + }, + { + name: 'sourceProcessName', + type: 'keyword', + description: "The name of the event's source process.", + }, + { + name: 'sourceServiceName', + type: 'keyword', + description: 'The service that is responsible for generating this event.', + }, + { + name: 'sourceTranslatedAddress', + type: 'ip', + description: + 'Identifies the translated source that the event refers to in an IP network.', + }, + { + name: 'sourceTranslatedPort', + type: 'long', + description: + 'A port number after being translated by, for example, a firewall. Valid port numbers are 0 to 65535.', + }, + { + name: 'sourceTranslatedZoneExternalID', + type: 'keyword', + description: '', + }, + { + name: 'sourceTranslatedZoneURI', + type: 'keyword', + description: + 'The URI for the Translated Zone that the destination asset has been assigned to in ArcSight.', + }, + { + name: 'sourceUserId', + type: 'keyword', + description: + 'Identifies the source user by ID. This is the user associated with the source of the event. For example, in UNIX, the root user is generally associated with user ID 0.', + }, + { + name: 'sourceUserName', + type: 'keyword', + description: + 'Identifies the source user by name. Email addresses are also mapped into the UserName fields. The sender is a candidate to put into this field.', + }, + { + name: 'sourceUserPrivileges', + type: 'keyword', + description: + 'The typical values are "Administrator", "User", and "Guest". It identifies the source user\'s privileges. In UNIX, for example, activity executed by the root user would be identified with "Administrator".', + }, + { + name: 'sourceZoneExternalID', + type: 'keyword', + description: '', + }, + { + name: 'sourceZoneURI', + type: 'keyword', + description: + 'The URI for the Zone that the source asset has been assigned to in ArcSight.', + }, + { + name: 'startTime', + type: 'date', + description: + 'The time when the activity the event referred to started. The format is MMM dd yyyy HH:mm:ss or milliseconds since epoch (Jan 1st 1970)', + }, + { + name: 'transportProtocol', + type: 'keyword', + description: + 'Identifies the Layer-4 protocol used. The possible values are protocols such as TCP or UDP.', + }, + { + name: 'type', + type: 'long', + description: + '0 means base event, 1 means aggregated, 2 means correlation, and 3 means action. This field can be omitted for base events (type 0).', + }, + { + name: 'categoryDeviceType', + type: 'keyword', + description: 'Device type. Examples - Proxy, IDS, Web Server', + }, + { + name: 'categoryObject', + type: 'keyword', + description: + 'Object that the event is about. For example it can be an operating sytem, database, file, etc.', + }, + { + name: 'categoryBehavior', + type: 'keyword', + description: + "Action or a behavior associated with an event. It's what is being done to the object.", + }, + { + name: 'categoryTechnique', + type: 'keyword', + description: 'Technique being used (e.g. /DoS).', + }, + { + name: 'categoryDeviceGroup', + type: 'keyword', + description: 'General device group like Firewall.', + }, + { + name: 'categorySignificance', + type: 'keyword', + description: 'Characterization of the importance of the event.', + }, + { + name: 'categoryOutcome', + type: 'keyword', + description: 'Outcome of the event (e.g. sucess, failure, or attempt).', + }, + { + name: 'managerReceiptTime', + type: 'date', + description: 'When the Arcsight ESM received the event.', + }, + ], + }, + ], + }, + { + name: 'source.service.name', + type: 'keyword', + description: 'Service that is the source of the event.', + }, + { + name: 'destination.service.name', + type: 'keyword', + description: 'Service that is the target of the event.', + }, + ], + }, ]; diff --git a/x-pack/legacy/plugins/siem/server/utils/beat_schema/8.0.0/packetbeat.ts b/x-pack/legacy/plugins/siem/server/utils/beat_schema/8.0.0/packetbeat.ts index 6002c9370210e..0be2e48fe4668 100644 --- a/x-pack/legacy/plugins/siem/server/utils/beat_schema/8.0.0/packetbeat.ts +++ b/x-pack/legacy/plugins/siem/server/utils/beat_schema/8.0.0/packetbeat.ts @@ -15,91 +15,129 @@ export const packetbeatSchema: Schema = [ { key: 'ecs', title: 'ECS', - description: 'ECS fields.', + description: 'ECS Fields.', fields: [ { name: '@timestamp', - type: 'date', level: 'core', required: true, - example: '2016-05-23T08:05:34.853Z', + type: 'date', description: - 'Date/time when the event originated. For log events this is the date/time when the event was generated, and not when it was read. Required field for all events.', - }, - { - name: 'tags', - level: 'core', - type: 'keyword', - example: '["production", "env2"]', - description: 'List of keywords used to tag each event.', + 'Date/time when the event originated.\n\nThis is the date/time extracted from the event, typically representing when\nthe event was generated by the source.\n\nIf the event source has no original timestamp, this value is typically populated\nby the first time the event was received by the pipeline.\n\nRequired field for all events.', + example: '2016-05-23T08:05:34.853Z', }, { name: 'labels', level: 'core', type: 'object', - example: { - env: 'production', - application: 'foo-bar', - }, + object_type: 'keyword', description: - 'Key/value pairs. Can be used to add meta information to events. Should not contain nested objects. All values are stored as keyword. Example: `docker` and `k8s` labels.', + 'Custom key/value pairs.\n\nCan be used to add meta information to events. Should not contain nested objects.\nAll values are stored as keyword.\n\nExample: `docker` and `k8s` labels.', + example: '{"application": "foo-bar", "env": "production"}', }, { name: 'message', level: 'core', type: 'text', - example: 'Hello World', description: - 'For log events the message field contains the log message. In other use cases the message field can be used to concatenate different values which are then freely searchable. If multiple messages exist, they can be combined into one message.', + 'For log events the message field contains the log message, optimized\nfor viewing in a log viewer.\n\nFor structured logs without an original message field, other fields can be concatenated\nto form a human-readable summary of the event.\n\nIf multiple messages exist, they can be combined into one message.', + example: 'Hello World', + }, + { + name: 'tags', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'List of keywords used to tag each event.', + example: '["production", "env2"]', }, { name: 'agent', title: 'Agent', group: 2, description: - 'The agent fields contain the data about the software entity, if any, that collects, detects, or observes events on a host, or takes measurements on a host. Examples include Beats. Agents may also run on observers. ECS agent.* fields shall be populated with details of the agent running on the host or observer where the event happened or the measurement was taken.', + 'The agent fields contain the data about the software entity, if\nany, that collects, detects, or observes events on a host, or takes measurements\non a host.\n\nExamples include Beats. Agents may also run on observers. ECS agent.* fields\nshall be populated with details of the agent running on the host or observer\nwhere the event happened or the measurement was taken.', footnote: - 'Examples: In the case of Beats for logs, the agent.name is filebeat. For APM, it is the agent running in the app/service. The agent information does not change if data is sent through queuing systems like Kafka, Redis, or processing systems such as Logstash or APM Server.', + 'Examples: In the case of Beats for logs, the agent.name is filebeat.\nFor APM, it is the agent running in the app/service. The agent information does\nnot change if data is sent through queuing systems like Kafka, Redis, or processing\nsystems such as Logstash or APM Server.', type: 'group', fields: [ { - name: 'version', + name: 'ephemeral_id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Ephemeral identifier of this agent (if one exists).\n\nThis id normally changes across restarts, but `agent.id` does not.', + example: '8a4f500f', + }, + { + name: 'id', level: 'core', type: 'keyword', - description: 'Version of the agent.', - example: '6.0.0-rc2', + ignore_above: 1024, + description: + 'Unique identifier of this agent (if one exists).\n\nExample: For Beats this would be beat.id.', + example: '8a4f500d', }, { name: 'name', level: 'core', type: 'keyword', + ignore_above: 1024, description: - 'Name of the agent. This is a name that can be given to an agent. This can be helpful if for example two Filebeat instances are running on the same host but a human readable separation is needed on which Filebeat instance data is coming from. If no name is given, the name is often left empty.', + 'Custom name of the agent.\n\nThis is a name that can be given to an agent. This can be helpful if for example\ntwo Filebeat instances are running on the same host but a human readable separation\nis needed on which Filebeat instance data is coming from.\n\nIf no name is given, the name is often left empty.', example: 'foo', }, { name: 'type', level: 'core', type: 'keyword', + ignore_above: 1024, description: - 'Type of the agent. The agent type stays always the same and should be given by the agent used. In case of Filebeat the agent would always be Filebeat also if two Filebeat instances are run on the same machine.', + 'Type of the agent.\n\nThe agent type stays always the same and should be given by the agent used.\nIn case of Filebeat the agent would always be Filebeat also if two Filebeat\ninstances are run on the same machine.', example: 'filebeat', }, { - name: 'id', + name: 'version', level: 'core', type: 'keyword', + ignore_above: 1024, + description: 'Version of the agent.', + example: '6.0.0-rc2', + }, + ], + }, + { + name: 'as', + title: 'Autonomous System', + group: 2, + description: + 'An autonomous system (AS) is a collection of connected Internet Protocol\n(IP) routing prefixes under the control of one or more network operators on\nbehalf of a single administrative entity or domain that presents a common, clearly\ndefined routing policy to the internet.', + type: 'group', + fields: [ + { + name: 'number', + level: 'extended', + type: 'long', description: - 'Unique identifier of this agent (if one exists). Example: For Beats this would be beat.id.', - example: '8a4f500d', + 'Unique number allocated to the autonomous system. The autonomous\nsystem number (ASN) uniquely identifies each network on the Internet.', + example: 15169, }, { - name: 'ephemeral_id', + name: 'organization.name', level: 'extended', type: 'keyword', - description: - 'Ephemeral identifier of this agent (if one exists). This id normally changes across restarts, but `agent.id` does not.', - example: '8a4f500f', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Organization name.', + example: 'Google LLC', }, ], }, @@ -108,2673 +146,6183 @@ export const packetbeatSchema: Schema = [ title: 'Client', group: 2, description: - 'A client is defined as the initiator of a network connection for events regarding sessions, connections, or bidirectional flow records. For TCP events, the client is the initiator of the TCP connection that sends the SYN packet(s). For other protocols, the client is generally the initiator or requestor in the network transaction. Some systems use the term "originator" to refer the client in TCP connections. The client fields describe details about the system acting as the client in the network event. Client fields are usually populated in conjunction with server fields. Client fields are generally not populated for packet-level events. Client / server representations can add semantic context to an exchange, which is helpful to visualize the data in certain situations. If your context falls in that category, you should still ensure that source and destination are filled appropriately.', + 'A client is defined as the initiator of a network connection for\nevents regarding sessions, connections, or bidirectional flow records.\n\nFor TCP events, the client is the initiator of the TCP connection that sends\nthe SYN packet(s). For other protocols, the client is generally the initiator\nor requestor in the network transaction. Some systems use the term "originator"\nto refer the client in TCP connections. The client fields describe details about\nthe system acting as the client in the network event. Client fields are usually\npopulated in conjunction with server fields. Client fields are generally not\npopulated for packet-level events.\n\nClient / server representations can add semantic context to an exchange, which\nis helpful to visualize the data in certain situations. If your context falls\nin that category, you should still ensure that source and destination are filled\nappropriately.', type: 'group', fields: [ { name: 'address', level: 'extended', type: 'keyword', + ignore_above: 1024, description: - 'Some event client addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is.', + 'Some event client addresses are defined ambiguously. The event\nwill sometimes list an IP, a domain or a unix socket. You should always store\nthe raw address in the `.address` field.\n\nThen it should be duplicated to `.ip` or `.domain`, depending on which one\nit is.', }, { - name: 'ip', - level: 'core', - type: 'ip', - description: 'IP address of the client. Can be one or multiple IPv4 or IPv6 addresses.', + name: 'as.number', + level: 'extended', + type: 'long', + description: + 'Unique number allocated to the autonomous system. The autonomous\nsystem number (ASN) uniquely identifies each network on the Internet.', + example: 15169, }, { - name: 'port', + name: 'as.organization.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Organization name.', + example: 'Google LLC', + }, + { + name: 'bytes', level: 'core', type: 'long', - description: 'Port of the client.', + format: 'bytes', + description: 'Bytes sent from the client to the server.', + example: 184, }, { - name: 'mac', + name: 'domain', level: 'core', type: 'keyword', - description: 'MAC address of the client.', + ignore_above: 1024, + description: 'Client domain.', }, { - name: 'domain', + name: 'geo.city_name', level: 'core', type: 'keyword', - description: 'Client domain.', + ignore_above: 1024, + description: 'City name.', + example: 'Montreal', }, { - name: 'bytes', + name: 'geo.continent_name', level: 'core', - type: 'long', - format: 'bytes', - example: 184, - description: 'Bytes sent from the client to the server.', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the continent.', + example: 'North America', }, { - name: 'packets', + name: 'geo.country_iso_code', level: 'core', - type: 'long', - example: 12, - description: 'Packets sent from the client to the server.', + type: 'keyword', + ignore_above: 1024, + description: 'Country ISO code.', + example: 'CA', }, { - name: 'geo', - title: 'Geo', - group: 2, - description: - 'Geo fields can carry data about a specific location related to an event or geo information derived from an IP field.', - type: 'group', - fields: [ - { - name: 'location', - level: 'core', - type: 'geo_point', - description: 'Longitude and latitude.', - example: '{ "lon": -73.614830, "lat": 45.505918 }', - }, - { - name: 'continent_name', - level: 'core', - type: 'keyword', - description: 'Name of the continent.', - example: 'North America', - }, - { - name: 'country_name', - level: 'core', - type: 'keyword', - description: 'Country name.', - example: 'Canada', - }, - { - name: 'region_name', - level: 'core', - type: 'keyword', - description: 'Region name.', - example: 'Quebec', - }, - { - name: 'city_name', - level: 'core', - type: 'keyword', - description: 'City name.', - example: 'Montreal', - }, - { - name: 'country_iso_code', - level: 'core', - type: 'keyword', - description: 'Country ISO code.', - example: 'CA', - }, - { - name: 'region_iso_code', - level: 'core', - type: 'keyword', - description: 'Region ISO code.', - example: 'CA-QC', - }, - { - name: 'name', - level: 'extended', - type: 'keyword', - description: - 'User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation.', - example: 'boston-dc', - }, - ], + name: 'geo.country_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Country name.', + example: 'Canada', }, - ], - }, - { - name: 'cloud', - title: 'Cloud', - group: 2, - description: 'Fields related to the cloud or infrastructure the events are coming from.', - footnote: - 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.', - type: 'group', - fields: [ { - name: 'provider', + name: 'geo.location', + level: 'core', + type: 'geo_point', + description: 'Longitude and latitude.', + example: '{ "lon": -73.614830, "lat": 45.505918 }', + }, + { + name: 'geo.name', level: 'extended', - example: 'ec2', type: 'keyword', + ignore_above: 1024, description: - 'Name of the cloud provider. Example values are ec2, gce, or digitalocean.', + 'User-defined description of a location, at the level of granularity\nthey care about.\n\nCould be the name of their data centers, the floor number, if this describes\na local physical entity, city names.\n\nNot typically used in automated geolocation.', + example: 'boston-dc', }, { - name: 'availability_zone', - level: 'extended', - example: 'us-east-1c', + name: 'geo.region_iso_code', + level: 'core', type: 'keyword', - description: 'Availability zone in which this host is running.', + ignore_above: 1024, + description: 'Region ISO code.', + example: 'CA-QC', }, { - name: 'region', - level: 'extended', + name: 'geo.region_name', + level: 'core', type: 'keyword', - example: 'us-east-1', - description: 'Region in which this host is running.', + ignore_above: 1024, + description: 'Region name.', + example: 'Quebec', }, { - name: 'instance.id', - level: 'extended', - type: 'keyword', - example: 'i-1234567890abcdef0', - description: 'Instance ID of the host machine.', + name: 'ip', + level: 'core', + type: 'ip', + description: + 'IP address of the client.\n\nCan be one or multiple IPv4 or IPv6 addresses.', }, { - name: 'instance.name', - level: 'extended', + name: 'mac', + level: 'core', type: 'keyword', - description: 'Instance name of the host machine.', + ignore_above: 1024, + description: 'MAC address of the client.', }, { - name: 'machine.type', + name: 'nat.ip', level: 'extended', - type: 'keyword', - example: 't2.medium', - description: 'Machine type of the host machine.', + type: 'ip', + description: + 'Translated IP of source based NAT sessions (e.g. internal client\nto internet).\n\nTypically connections traversing load balancers, firewalls, or routers.', }, { - name: 'account.id', + name: 'nat.port', level: 'extended', - type: 'keyword', - example: 666777888999, + type: 'long', + format: 'string', description: - 'The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.', + 'Translated port of source based NAT sessions (e.g. internal client\nto internet).\n\nTypically connections traversing load balancers, firewalls, or routers.', }, - ], - }, - { - name: 'container', - title: 'Container', - group: 2, - description: - 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.', - type: 'group', - fields: [ { - name: 'runtime', - level: 'extended', - type: 'keyword', - description: 'Runtime managing this container.', - example: 'docker', + name: 'packets', + level: 'core', + type: 'long', + description: 'Packets sent from the client to the server.', + example: 12, }, { - name: 'id', + name: 'port', level: 'core', + type: 'long', + format: 'string', + description: 'Port of the client.', + }, + { + name: 'registered_domain', + level: 'extended', type: 'keyword', - description: 'Unique container id.', + ignore_above: 1024, + description: + 'The highest registered client domain, stripped of the subdomain.\n\nFor example, the registered domain for "foo.google.com" is "google.com".\n\nThis value can be determined precisely with a list like the public suffix\nlist (http://publicsuffix.org). Trying to approximate this by simply taking\nthe last two labels will not work well for TLDs such as "co.uk".', + example: 'google.com', }, { - name: 'image.name', + name: 'top_level_domain', level: 'extended', type: 'keyword', - description: 'Name of the image the container was built on.', + ignore_above: 1024, + description: + 'The effective top level domain (eTLD), also known as the domain\nsuffix, is the last part of the domain name. For example, the top level domain\nfor google.com is "com".\n\nThis value can be determined precisely with a list like the public suffix\nlist (http://publicsuffix.org). Trying to approximate this by simply taking\nthe last label will not work well for effective TLDs such as "co.uk".', + example: 'co.uk', }, { - name: 'image.tag', + name: 'user.domain', level: 'extended', type: 'keyword', - description: 'Container image tag.', + ignore_above: 1024, + description: + 'Name of the directory the user is a member of.\n\nFor example, an LDAP or Active Directory domain name.', }, { - name: 'name', + name: 'user.email', level: 'extended', type: 'keyword', - description: 'Container name.', + ignore_above: 1024, + description: 'User email address.', }, { - name: 'labels', + name: 'user.full_name', level: 'extended', - type: 'object', - object_type: 'keyword', - description: 'Image labels.', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: "User's full name, if available.", + example: 'Albert Einstein', }, - ], - }, - { - name: 'destination', - title: 'Destination', - group: 2, - description: - 'Destination fields describe details about the destination of a packet/event. Destination fields are usually populated in conjunction with source fields.', - type: 'group', - fields: [ { - name: 'address', + name: 'user.group.domain', level: 'extended', type: 'keyword', + ignore_above: 1024, description: - 'Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is.', + 'Name of the directory the group is a member of.\n\nFor example, an LDAP or Active Directory domain name.', }, { - name: 'ip', - level: 'core', - type: 'ip', - description: - 'IP address of the destination. Can be one or multiple IPv4 or IPv6 addresses.', + name: 'user.group.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifier for the group on the system/platform.', }, { - name: 'port', - level: 'core', - type: 'long', - description: 'Port of the destination.', + name: 'user.group.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the group.', }, { - name: 'mac', - level: 'core', + name: 'user.hash', + level: 'extended', type: 'keyword', - description: 'MAC address of the destination.', + ignore_above: 1024, + description: + 'Unique user hash to correlate information for a user in anonymized\nform.\n\nUseful if `user.id` or `user.name` contain confidential information and cannot\nbe used.', }, { - name: 'domain', + name: 'user.id', level: 'core', type: 'keyword', - description: 'Destination domain.', + ignore_above: 1024, + description: 'Unique identifiers of the user.', }, { - name: 'bytes', + name: 'user.name', level: 'core', - type: 'long', - format: 'bytes', - example: 184, - description: 'Bytes sent from the destination to the source.', - }, - { - name: 'packets', - level: 'core', - type: 'long', - example: 12, - description: 'Packets sent from the destination to the source.', - }, - { - name: 'geo', - title: 'Geo', - group: 2, - description: - 'Geo fields can carry data about a specific location related to an event or geo information derived from an IP field.', - type: 'group', - fields: [ - { - name: 'location', - level: 'core', - type: 'geo_point', - description: 'Longitude and latitude.', - example: '{ "lon": -73.614830, "lat": 45.505918 }', - }, - { - name: 'continent_name', - level: 'core', - type: 'keyword', - description: 'Name of the continent.', - example: 'North America', - }, - { - name: 'country_name', - level: 'core', - type: 'keyword', - description: 'Country name.', - example: 'Canada', - }, - { - name: 'region_name', - level: 'core', - type: 'keyword', - description: 'Region name.', - example: 'Quebec', - }, - { - name: 'city_name', - level: 'core', - type: 'keyword', - description: 'City name.', - example: 'Montreal', - }, - { - name: 'country_iso_code', - level: 'core', - type: 'keyword', - description: 'Country ISO code.', - example: 'CA', - }, - { - name: 'region_iso_code', - level: 'core', - type: 'keyword', - description: 'Region ISO code.', - example: 'CA-QC', - }, + type: 'keyword', + ignore_above: 1024, + multi_fields: [ { - name: 'name', - level: 'extended', - type: 'keyword', - description: - 'User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation.', - example: 'boston-dc', + name: 'text', + type: 'text', + norms: false, + default_field: false, }, ], + description: 'Short name or login of the user.', + example: 'albert', }, ], }, { - name: 'ecs', - title: 'ECS', + name: 'cloud', + title: 'Cloud', group: 2, - description: 'Meta-information specific to ECS.', + description: 'Fields related to the cloud or infrastructure the events are coming\nfrom.', + footnote: + 'Examples: If Metricbeat is running on an EC2 host and fetches data\nfrom its host, the cloud info contains the data about this machine. If Metricbeat\nruns on a remote machine outside the cloud and fetches data from a service running\nin the cloud, the field contains cloud data from the machine the service is\nrunning on.', type: 'group', fields: [ { - name: 'version', - level: 'core', + name: 'account.id', + level: 'extended', type: 'keyword', - required: true, + ignore_above: 1024, description: - 'ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. The current version is 1.0.0-beta2 .', - example: '1.0.0-beta2', + 'The cloud account or organization id used to identify different\nentities in a multi-tenant environment.\n\nExamples: AWS account id, Google Cloud ORG Id, or other unique identifier.', + example: 666777888999, }, - ], - }, - { - name: 'error', - title: 'Error', - group: 2, - description: - 'These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error.', - type: 'group', - fields: [ { - name: 'id', - level: 'core', + name: 'availability_zone', + level: 'extended', type: 'keyword', - description: 'Unique identifier for the error.', + ignore_above: 1024, + description: 'Availability zone in which this host is running.', + example: 'us-east-1c', }, { - name: 'message', - level: 'core', - type: 'text', - description: 'Error message.', + name: 'instance.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Instance ID of the host machine.', + example: 'i-1234567890abcdef0', }, { - name: 'code', - level: 'core', + name: 'instance.name', + level: 'extended', type: 'keyword', - description: 'Error code describing the error.', + ignore_above: 1024, + description: 'Instance name of the host machine.', }, - ], - }, - { - name: 'event', - title: 'Event', - group: 2, - description: - 'The event fields are used for context information about the log or metric event itself. A log is defined as an event containing details of something that happened. Log events must include the time at which the thing happened. Examples of log events include a process starting on a host, a network packet being sent from a source to a destination, or a network connection between a client and a server being initiated or closed. A metric is defined as an event containing one or more numerical or categorical measurements and the time at which the measurement was taken. Examples of metric events include memory pressure measured on a host, or vulnerabilities measured on a scanned host.', - type: 'group', - fields: [ { - name: 'id', - level: 'core', + name: 'machine.type', + level: 'extended', type: 'keyword', - description: 'Unique ID to describe the event.', - example: '8a4f500d', + ignore_above: 1024, + description: 'Machine type of the host machine.', + example: 't2.medium', }, { - name: 'kind', + name: 'provider', level: 'extended', type: 'keyword', + ignore_above: 1024, description: - 'The kind of the event. This gives information about what type of information the event contains, without being specific to the contents of the event. Examples are `event`, `state`, `alarm`. Warning: In future versions of ECS, we plan to provide a list of acceptable values for this field, please use with caution.', - example: 'state', + 'Name of the cloud provider. Example values are aws, azure, gcp,\nor digitalocean.', + example: 'aws', }, { - name: 'category', - level: 'core', + name: 'region', + level: 'extended', type: 'keyword', - description: - 'Event category. This contains high-level information about the contents of the event. It is more generic than `event.action`, in the sense that typically a category contains multiple actions. Warning: In future versions of ECS, we plan to provide a list of acceptable values for this field, please use with caution.', - example: 'user-management', + ignore_above: 1024, + description: 'Region in which this host is running.', + example: 'us-east-1', }, + ], + }, + { + name: 'code_signature', + title: 'Code Signature', + group: 2, + description: 'These fields contain information about binary code signatures.', + type: 'group', + fields: [ { - name: 'action', + name: 'exists', level: 'core', - type: 'keyword', - description: - 'The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer.', - example: 'user-password-change', + type: 'boolean', + description: 'Boolean to capture if a signature is present.', + example: 'true', + default_field: false, }, { - name: 'outcome', + name: 'status', level: 'extended', type: 'keyword', + ignore_above: 1024, description: - 'The outcome of the event. If the event describes an action, this fields contains the outcome of that action. Examples outcomes are `success` and `failure`. Warning: In future versions of ECS, we plan to provide a list of acceptable values for this field, please use with caution.', - example: 'success', - }, - { - name: 'type', - level: 'core', - type: 'keyword', - description: 'Reserved for future usage. Please avoid using this field for user data.', + 'Additional information about the certificate status.\n\nThis is useful for logging cryptographic errors with the certificate validity\nor trust status. Leave unpopulated if the validity or trust of the certificate\nwas unchecked.', + example: 'ERROR_UNTRUSTED_ROOT', + default_field: false, }, { - name: 'module', + name: 'subject_name', level: 'core', type: 'keyword', - description: - 'Name of the module this data is coming from. This information is coming from the modules used in Beats or Logstash.', - example: 'mysql', + ignore_above: 1024, + description: 'Subject name of the code signer', + example: 'Microsoft Corporation', + default_field: false, }, { - name: 'dataset', - level: 'core', - type: 'keyword', + name: 'trusted', + level: 'extended', + type: 'boolean', description: - 'Name of the dataset. The concept of a `dataset` (fileset / metricset) is used in Beats as a subset of modules. It contains the information which is currently stored in metricset.name and metricset.module or fileset.name.', - example: 'stats', + 'Stores the trust status of the certificate chain.\n\nValidating the trust of the certificate chain may be complicated, and this\nfield should only be populated by tools that actively check the status.', + example: 'true', + default_field: false, }, { - name: 'severity', - level: 'core', - type: 'long', - example: '7', + name: 'valid', + level: 'extended', + type: 'boolean', description: - "Severity describes the severity of the event. What the different severity values mean can very different between use cases. It's up to the implementer to make sure severities are consistent across events. ", + 'Boolean to capture if the digital signature is verified against\nthe binary content.\n\nLeave unpopulated if a certificate was unchecked.', + example: 'true', + default_field: false, }, + ], + }, + { + name: 'container', + title: 'Container', + group: 2, + description: + 'Container fields are used for meta information about the specific\ncontainer that is the source of information.\n\nThese fields help correlate data based containers from any runtime.', + type: 'group', + fields: [ { - name: 'original', + name: 'id', level: 'core', type: 'keyword', - example: - 'Sep 19 08:26:10 host CEF:0|Security| threatmanager|1.0|100| worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2spt=1232', - description: - 'Raw text message of entire event. Used to demonstrate log integrity. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`.', - index: false, - doc_values: false, + ignore_above: 1024, + description: 'Unique container id.', }, { - name: 'hash', + name: 'image.name', level: 'extended', type: 'keyword', - example: '123456789012345678901234567890ABCD', - description: - 'Hash (perhaps logstash fingerprint) of raw field to be able to demonstrate log integrity.', - }, - { - name: 'duration', - level: 'core', - type: 'long', - format: 'duration', - input_format: 'nanoseconds', - description: - 'Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time.', + ignore_above: 1024, + description: 'Name of the image the container was built on.', }, { - name: 'timezone', + name: 'image.tag', level: 'extended', type: 'keyword', - description: - 'This field should be populated when the event\'s timestamp does not include timezone information already (e.g. default Syslog timestamps). It\'s optional otherwise. Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00").', + ignore_above: 1024, + description: 'Container image tags.', }, { - name: 'created', - level: 'core', - type: 'date', - description: - 'event.created contains the date when the event was created. This timestamp is distinct from @timestamp in that @timestamp contains the processed timestamp. For logs these two timestamps can be different as the timestamp in the log line and when the event is read for example by Filebeat are not identical. `@timestamp` must contain the timestamp extracted from the log line, event.created when the log line is read. The same could apply to package capturing where @timestamp contains the timestamp extracted from the network package and event.created when the event was created. In case the two timestamps are identical, @timestamp should be used.', - }, - { - name: 'start', + name: 'labels', level: 'extended', - type: 'date', - description: - 'event.start contains the date when the event started or when the activity was first observed.', + type: 'object', + object_type: 'keyword', + description: 'Image labels.', }, { - name: 'end', + name: 'name', level: 'extended', - type: 'date', - description: - 'event.end contains the date when the event ended or when the activity was last observed.', - }, - { - name: 'risk_score', - level: 'core', - type: 'float', - description: - "Risk score or priority of the event (e.g. security solutions). Use your system's original value here. ", + type: 'keyword', + ignore_above: 1024, + description: 'Container name.', }, { - name: 'risk_score_norm', + name: 'runtime', level: 'extended', - type: 'float', - description: - 'Normalized risk score or priority of the event, on a scale of 0 to 100. This is mainly useful if you use more than one system that assigns risk scores, and you want to see a normalized value across all systems.', + type: 'keyword', + ignore_above: 1024, + description: 'Runtime managing this container.', + example: 'docker', }, ], }, { - name: 'file', + name: 'destination', + title: 'Destination', group: 2, - title: 'File', description: - 'A file is defined as a set of information that has been created on, or has existed on a filesystem. File objects can be associated with host events, network events, and/or file events (e.g., those produced by File Integrity Monitoring [FIM] products or services). File fields provide details about the affected file associated with the event or metric.', + 'Destination fields describe details about the destination of a packet/event.\n\nDestination fields are usually populated in conjunction with source fields.', type: 'group', fields: [ { - name: 'path', + name: 'address', level: 'extended', type: 'keyword', - description: 'Path to the file.', + ignore_above: 1024, + description: + 'Some event destination addresses are defined ambiguously. The\nevent will sometimes list an IP, a domain or a unix socket. You should always\nstore the raw address in the `.address` field.\n\nThen it should be duplicated to `.ip` or `.domain`, depending on which one\nit is.', }, { - name: 'target_path', + name: 'as.number', level: 'extended', - type: 'keyword', - description: 'Target path for symlinks.', + type: 'long', + description: + 'Unique number allocated to the autonomous system. The autonomous\nsystem number (ASN) uniquely identifies each network on the Internet.', + example: 15169, }, { - name: 'extension', + name: 'as.organization.name', level: 'extended', type: 'keyword', - description: 'File extension. This should allow easy filtering by file extensions.', - example: 'png', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Organization name.', + example: 'Google LLC', }, { - name: 'type', - level: 'extended', - type: 'keyword', - description: 'File type (file, dir, or symlink).', + name: 'bytes', + level: 'core', + type: 'long', + format: 'bytes', + description: 'Bytes sent from the destination to the source.', + example: 184, }, { - name: 'device', - level: 'extended', + name: 'domain', + level: 'core', type: 'keyword', - description: 'Device that is the source of the file.', + ignore_above: 1024, + description: 'Destination domain.', }, { - name: 'inode', - level: 'extended', + name: 'geo.city_name', + level: 'core', type: 'keyword', - description: 'Inode representing the file in the filesystem.', + ignore_above: 1024, + description: 'City name.', + example: 'Montreal', }, { - name: 'uid', - level: 'extended', + name: 'geo.continent_name', + level: 'core', type: 'keyword', - description: 'The user ID (UID) or security identifier (SID) of the file owner.', + ignore_above: 1024, + description: 'Name of the continent.', + example: 'North America', }, { - name: 'owner', - level: 'extended', + name: 'geo.country_iso_code', + level: 'core', type: 'keyword', - description: "File owner's username.", + ignore_above: 1024, + description: 'Country ISO code.', + example: 'CA', }, { - name: 'gid', - level: 'extended', + name: 'geo.country_name', + level: 'core', type: 'keyword', - description: 'Primary group ID (GID) of the file.', + ignore_above: 1024, + description: 'Country name.', + example: 'Canada', }, { - name: 'group', - level: 'extended', - type: 'keyword', - description: 'Primary group name of the file.', + name: 'geo.location', + level: 'core', + type: 'geo_point', + description: 'Longitude and latitude.', + example: '{ "lon": -73.614830, "lat": 45.505918 }', }, { - name: 'mode', - level: 'extended', - type: 'keyword', - example: 416, - description: 'Mode of the file in octal representation.', - }, - { - name: 'size', - level: 'extended', - type: 'long', - format: 'bytes', - description: 'File size in bytes (field is only added when `type` is `file`).', - }, - { - name: 'mtime', - level: 'extended', - type: 'date', - description: 'Last time file content was modified.', - }, - { - name: 'ctime', - level: 'extended', - type: 'date', - description: 'Last time file metadata changed.', - }, - ], - }, - { - name: 'group', - title: 'Group', - group: 2, - description: - 'The group fields are meant to represent groups that are relevant to the event.', - type: 'group', - fields: [ - { - name: 'id', + name: 'geo.name', level: 'extended', type: 'keyword', - description: 'Unique identifier for the group on the system/platform.', - }, - { - name: 'name', - level: 'extended', - type: 'keyword', - description: 'Name of the group.', - }, - ], - }, - { - name: 'host', - title: 'Host', - group: 2, - description: - 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or on which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.', - type: 'group', - fields: [ - { - name: 'hostname', - level: 'core', - type: 'keyword', + ignore_above: 1024, description: - 'Hostname of the host. It normally contains what the `hostname` command returns on the host machine.', + 'User-defined description of a location, at the level of granularity\nthey care about.\n\nCould be the name of their data centers, the floor number, if this describes\na local physical entity, city names.\n\nNot typically used in automated geolocation.', + example: 'boston-dc', }, { - name: 'name', + name: 'geo.region_iso_code', level: 'core', type: 'keyword', - description: - 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.', + ignore_above: 1024, + description: 'Region ISO code.', + example: 'CA-QC', }, { - name: 'id', + name: 'geo.region_name', level: 'core', type: 'keyword', - description: - 'Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.', + ignore_above: 1024, + description: 'Region name.', + example: 'Quebec', }, { name: 'ip', level: 'core', type: 'ip', - description: 'Host ip address.', + description: + 'IP address of the destination.\n\nCan be one or multiple IPv4 or IPv6 addresses.', }, { name: 'mac', level: 'core', type: 'keyword', - description: 'Host mac address.', + ignore_above: 1024, + description: 'MAC address of the destination.', }, { - name: 'type', - level: 'core', - type: 'keyword', + name: 'nat.ip', + level: 'extended', + type: 'ip', description: - 'Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.', + 'Translated ip of destination based NAT sessions (e.g. internet\nto private DMZ)\n\nTypically used with load balancers, firewalls, or routers.', }, { - name: 'architecture', - level: 'core', - type: 'keyword', - example: 'x86_64', - description: 'Operating system architecture.', + name: 'nat.port', + level: 'extended', + type: 'long', + format: 'string', + description: + 'Port the source session is translated to by NAT Device.\n\nTypically used with load balancers, firewalls, or routers.', }, { - name: 'os', - title: 'Operating System', - group: 2, - description: 'The OS fields contain information about the operating system.', - reusable: { - top_level: false, - expected: ['observer', 'host', 'user_agent'], - }, - type: 'group', - fields: [ - { - name: 'platform', - level: 'extended', - type: 'keyword', - description: 'Operating system platform (such centos, ubuntu, windows).', - example: 'darwin', - }, - { - name: 'name', - level: 'extended', - type: 'keyword', - example: 'Mac OS X', - description: 'Operating system name, without the version.', - }, - { - name: 'full', - level: 'extended', - type: 'keyword', - example: 'Mac OS Mojave', - description: 'Operating system name, including the version or code name.', - }, - { - name: 'family', - level: 'extended', - type: 'keyword', - example: 'debian', - description: 'OS family (such as redhat, debian, freebsd, windows).', - }, - { - name: 'version', - level: 'extended', - type: 'keyword', - example: '10.14.1', - description: 'Operating system version as a raw string.', - }, - { - name: 'kernel', - level: 'extended', - type: 'keyword', - example: '4.4.0-112-generic', - description: 'Operating system kernel version as a raw string.', - }, - ], + name: 'packets', + level: 'core', + type: 'long', + description: 'Packets sent from the destination to the source.', + example: 12, }, { - name: 'geo', - title: 'Geo', - group: 2, - description: - 'Geo fields can carry data about a specific location related to an event or geo information derived from an IP field.', - type: 'group', - fields: [ - { - name: 'location', - level: 'core', - type: 'geo_point', - description: 'Longitude and latitude.', - example: '{ "lon": -73.614830, "lat": 45.505918 }', - }, - { - name: 'continent_name', - level: 'core', - type: 'keyword', - description: 'Name of the continent.', - example: 'North America', - }, - { - name: 'country_name', - level: 'core', - type: 'keyword', - description: 'Country name.', - example: 'Canada', - }, - { - name: 'region_name', - level: 'core', - type: 'keyword', - description: 'Region name.', - example: 'Quebec', - }, - { - name: 'city_name', - level: 'core', - type: 'keyword', - description: 'City name.', - example: 'Montreal', - }, - { - name: 'country_iso_code', - level: 'core', - type: 'keyword', - description: 'Country ISO code.', - example: 'CA', - }, - { - name: 'region_iso_code', - level: 'core', - type: 'keyword', - description: 'Region ISO code.', - example: 'CA-QC', - }, - { - name: 'name', - level: 'extended', - type: 'keyword', - description: - 'User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation.', - example: 'boston-dc', - }, - ], + name: 'port', + level: 'core', + type: 'long', + format: 'string', + description: 'Port of the destination.', }, - ], - }, - { - name: 'http', - title: 'HTTP', - group: 2, - description: 'Fields related to HTTP activity.', - type: 'group', - fields: [ { - name: 'request.method', + name: 'registered_domain', level: 'extended', type: 'keyword', + ignore_above: 1024, description: - 'Http request method. The field value must be normalized to lowercase for querying. See "Lowercase Capitalization" in the "Implementing ECS" section.', - example: 'get, post, put', + 'The highest registered destination domain, stripped of the subdomain.\n\nFor example, the registered domain for "foo.google.com" is "google.com".\n\nThis value can be determined precisely with a list like the public suffix\nlist (http://publicsuffix.org). Trying to approximate this by simply taking\nthe last two labels will not work well for TLDs such as "co.uk".', + example: 'google.com', }, { - name: 'request.body.content', + name: 'top_level_domain', level: 'extended', type: 'keyword', - description: 'The full http request body.', - example: 'Hello world', + ignore_above: 1024, + description: + 'The effective top level domain (eTLD), also known as the domain\nsuffix, is the last part of the domain name. For example, the top level domain\nfor google.com is "com".\n\nThis value can be determined precisely with a list like the public suffix\nlist (http://publicsuffix.org). Trying to approximate this by simply taking\nthe last label will not work well for effective TLDs such as "co.uk".', + example: 'co.uk', }, { - name: 'request.referrer', + name: 'user.domain', level: 'extended', type: 'keyword', - description: 'Referrer for this HTTP request.', - example: 'https://blog.example.com/', - }, - { - name: 'response.status_code', - level: 'extended', - type: 'long', - description: 'Http response status code.', - example: 404, + ignore_above: 1024, + description: + 'Name of the directory the user is a member of.\n\nFor example, an LDAP or Active Directory domain name.', }, { - name: 'response.body.content', + name: 'user.email', level: 'extended', type: 'keyword', - description: 'The full http response body.', - example: 'Hello world', + ignore_above: 1024, + description: 'User email address.', }, { - name: 'version', + name: 'user.full_name', level: 'extended', type: 'keyword', - description: 'Http version.', - example: 1.1, + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: "User's full name, if available.", + example: 'Albert Einstein', }, { - name: 'request.bytes', + name: 'user.group.domain', level: 'extended', - type: 'long', - format: 'bytes', - description: 'Total size in bytes of the request (body and headers).', - example: 1437, + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the directory the group is a member of.\n\nFor example, an LDAP or Active Directory domain name.', }, { - name: 'request.body.bytes', + name: 'user.group.id', level: 'extended', - type: 'long', - format: 'bytes', - description: 'Size in bytes of the request body.', - example: 887, + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifier for the group on the system/platform.', }, { - name: 'response.bytes', + name: 'user.group.name', level: 'extended', - type: 'long', - format: 'bytes', - description: 'Total size in bytes of the response (body and headers).', - example: 1437, + type: 'keyword', + ignore_above: 1024, + description: 'Name of the group.', }, { - name: 'response.body.bytes', + name: 'user.hash', level: 'extended', - type: 'long', - format: 'bytes', - description: 'Size in bytes of the response body.', - example: 887, + type: 'keyword', + ignore_above: 1024, + description: + 'Unique user hash to correlate information for a user in anonymized\nform.\n\nUseful if `user.id` or `user.name` contain confidential information and cannot\nbe used.', }, - ], - }, - { - name: 'log', - title: 'Log', - description: 'Fields which are specific to log events.', - type: 'group', - fields: [ { - name: 'level', + name: 'user.id', level: 'core', type: 'keyword', - description: 'Log level of the log event. Some examples are `WARN`, `ERR`, `INFO`.', - example: 'ERR', + ignore_above: 1024, + description: 'Unique identifiers of the user.', }, { - name: 'original', + name: 'user.name', level: 'core', type: 'keyword', - example: 'Sep 19 08:26:10 localhost My log', - index: false, - doc_values: false, - description: - " This is the original log message and contains the full log message before splitting it up in multiple parts. In contrast to the `message` field which can contain an extracted part of the log message, this field contains the original, full log message. It can have already some modifications applied like encoding or new lines removed to clean up the log message. This field is not indexed and doc_values are disabled so it can't be queried but the value can be retrieved from `_source`. ", + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Short name or login of the user.', + example: 'albert', }, ], }, { - name: 'network', - title: 'Network', + name: 'dll', + title: 'DLL', group: 2, description: - 'The network is defined as the communication path over which a host or network event happens. The network.* fields should be populated with details about the network activity associated with an event.', + 'These fields contain information about code libraries dynamically\nloaded into processes.\n\n\nMany operating systems refer to "shared code libraries" with different names,\nbut this field set refers to all of the following:\n\n* Dynamic-link library (`.dll`) commonly used on Windows\n\n* Shared Object (`.so`) commonly used on Unix-like operating systems\n\n* Dynamic library (`.dylib`) commonly used on macOS', type: 'group', fields: [ { - name: 'name', + name: 'code_signature.exists', + level: 'core', + type: 'boolean', + description: 'Boolean to capture if a signature is present.', + example: 'true', + default_field: false, + }, + { + name: 'code_signature.status', level: 'extended', type: 'keyword', - description: 'Name given by operators to sections of their network.', - example: 'Guest Wifi', + ignore_above: 1024, + description: + 'Additional information about the certificate status.\n\nThis is useful for logging cryptographic errors with the certificate validity\nor trust status. Leave unpopulated if the validity or trust of the certificate\nwas unchecked.', + example: 'ERROR_UNTRUSTED_ROOT', + default_field: false, }, { - name: 'type', + name: 'code_signature.subject_name', level: 'core', type: 'keyword', + ignore_above: 1024, + description: 'Subject name of the code signer', + example: 'Microsoft Corporation', + default_field: false, + }, + { + name: 'code_signature.trusted', + level: 'extended', + type: 'boolean', description: - 'In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc The field value must be normalized to lowercase for querying. See "Lowercase Capitalization" in the "Implementing ECS" section.', - example: 'ipv4', + 'Stores the trust status of the certificate chain.\n\nValidating the trust of the certificate chain may be complicated, and this\nfield should only be populated by tools that actively check the status.', + example: 'true', + default_field: false, }, { - name: 'iana_number', + name: 'code_signature.valid', level: 'extended', - type: 'keyword', + type: 'boolean', description: - 'IANA Protocol Number (https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml). Standardized list of protocols. This aligns well with NetFlow and sFlow related logs which use the IANA Protocol Number.', - example: 6, + 'Boolean to capture if the digital signature is verified against\nthe binary content.\n\nLeave unpopulated if a certificate was unchecked.', + example: 'true', + default_field: false, }, { - name: 'transport', - level: 'core', + name: 'hash.md5', + level: 'extended', type: 'keyword', - description: - 'Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. See "Lowercase Capitalization" in the "Implementing ECS" section.', - example: 'tcp', + ignore_above: 1024, + description: 'MD5 hash.', + default_field: false, }, { - name: 'application', + name: 'hash.sha1', level: 'extended', type: 'keyword', - description: - 'A name given to an application. This can be arbitrarily assigned for things like microservices, but also apply to things like skype, icq, facebook, twitter. This would be used in situations where the vendor or service can be decoded such as from the source/dest IP owners, ports, or wire format. The field value must be normalized to lowercase for querying. See "Lowercase Capitalization" in the "Implementing ECS" section.', - example: 'aim', + ignore_above: 1024, + description: 'SHA1 hash.', + default_field: false, }, { - name: 'protocol', - level: 'core', + name: 'hash.sha256', + level: 'extended', type: 'keyword', - description: - 'L7 Network protocol name. ex. http, lumberjack, transport protocol. The field value must be normalized to lowercase for querying. See "Lowercase Capitalization" in the "Implementing ECS" section.', - example: 'http', + ignore_above: 1024, + description: 'SHA256 hash.', + default_field: false, }, { - name: 'direction', - level: 'core', + name: 'hash.sha512', + level: 'extended', type: 'keyword', - description: - "Direction of the network traffic. Recommended values are: * inbound * outbound * internal * external * unknown When mapping events from a host-based monitoring context, populate this field from the host's point of view. When mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of your network perimeter. ", - example: 'inbound', + ignore_above: 1024, + description: 'SHA512 hash.', + default_field: false, }, { - name: 'forwarded_ip', + name: 'name', level: 'core', - type: 'ip', - description: 'Host IP address when the source IP address is the proxy.', - example: '192.1.1.2', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the library.\n\nThis generally maps to the name of the file on disk.', + example: 'kernel32.dll', + default_field: false, }, { - name: 'community_id', + name: 'path', level: 'extended', type: 'keyword', - description: - 'A hash of source and destination IPs and ports, as well as the protocol used in a communication. This is a tool-agnostic standard to identify flows. Learn more at https://github.com/corelight/community-id-spec.', - example: '1:hO+sN4H+MG5MY/8hIrXPqc4ZQz0=', - }, - { - name: 'bytes', - level: 'core', - type: 'long', - format: 'bytes', - description: - 'Total bytes transferred in both directions. If `source.bytes` and `destination.bytes` are known, `network.bytes` is their sum.', - example: 368, - }, - { - name: 'packets', - level: 'core', - type: 'long', - description: - 'Total packets transferred in both directions. If `source.packets` and `destination.packets` are known, `network.packets` is their sum.', - example: 24, - }, - ], - }, - { - name: 'observer', - title: 'Observer', - group: 2, - description: - 'An observer is defined as a special network, security, or application device used to detect, observe, or create network, security, or application-related events and metrics. This could be a custom hardware appliance or a server that has been configured to run special network, security, or application software. Examples include firewalls, intrusion detection/prevention systems, network monitoring sensors, web application firewalls, data loss prevention systems, and APM servers. The observer.* fields shall be populated with details of the system, if any, that detects, observes and/or creates a network, security, or application event or metric. Message queues and ETL components used in processing events or metrics are not considered observers in ECS.', - type: 'group', - fields: [ - { - name: 'mac', - level: 'core', - type: 'keyword', - description: 'MAC address of the observer', - }, - { - name: 'ip', - level: 'core', - type: 'ip', - description: 'IP address of the observer.', + ignore_above: 1024, + description: 'Full file path of the library.', + example: 'C:\\Windows\\System32\\kernel32.dll', + default_field: false, }, { - name: 'hostname', - level: 'core', + name: 'pe.company', + level: 'extended', type: 'keyword', - description: 'Hostname of the observer.', + ignore_above: 1024, + description: 'Internal company name of the file, provided at compile-time.', + example: 'Microsoft Corporation', + default_field: false, }, { - name: 'vendor', - level: 'core', + name: 'pe.description', + level: 'extended', type: 'keyword', - description: 'observer vendor information.', + ignore_above: 1024, + description: 'Internal description of the file, provided at compile-time.', + example: 'Paint', + default_field: false, }, { - name: 'version', - level: 'core', + name: 'pe.file_version', + level: 'extended', type: 'keyword', - description: 'Observer version.', + ignore_above: 1024, + description: 'Internal version of the file, provided at compile-time.', + example: '6.3.9600.17415', + default_field: false, }, { - name: 'serial_number', + name: 'pe.original_file_name', level: 'extended', type: 'keyword', - description: 'Observer serial number.', + ignore_above: 1024, + description: 'Internal name of the file, provided at compile-time.', + example: 'MSPAINT.EXE', + default_field: false, }, { - name: 'type', - level: 'core', + name: 'pe.product', + level: 'extended', type: 'keyword', - description: - 'The type of the observer the data is coming from. There is no predefined list of observer types. Some examples are `forwarder`, `firewall`, `ids`, `ips`, `proxy`, `poller`, `sensor`, `APM server`.', - example: 'firewall', - }, - { - name: 'os', - title: 'Operating System', - group: 2, - description: 'The OS fields contain information about the operating system.', - reusable: { - top_level: false, - expected: ['observer', 'host', 'user_agent'], - }, - type: 'group', - fields: [ - { - name: 'platform', - level: 'extended', - type: 'keyword', - description: 'Operating system platform (such centos, ubuntu, windows).', - example: 'darwin', - }, - { - name: 'name', - level: 'extended', - type: 'keyword', - example: 'Mac OS X', - description: 'Operating system name, without the version.', - }, - { - name: 'full', - level: 'extended', - type: 'keyword', - example: 'Mac OS Mojave', - description: 'Operating system name, including the version or code name.', - }, - { - name: 'family', - level: 'extended', - type: 'keyword', - example: 'debian', - description: 'OS family (such as redhat, debian, freebsd, windows).', - }, - { - name: 'version', - level: 'extended', - type: 'keyword', - example: '10.14.1', - description: 'Operating system version as a raw string.', - }, - { - name: 'kernel', - level: 'extended', - type: 'keyword', - example: '4.4.0-112-generic', - description: 'Operating system kernel version as a raw string.', - }, - ], - }, - { - name: 'geo', - title: 'Geo', - group: 2, - description: - 'Geo fields can carry data about a specific location related to an event or geo information derived from an IP field.', - type: 'group', - fields: [ - { - name: 'location', - level: 'core', - type: 'geo_point', - description: 'Longitude and latitude.', - example: '{ "lon": -73.614830, "lat": 45.505918 }', - }, - { - name: 'continent_name', - level: 'core', - type: 'keyword', - description: 'Name of the continent.', - example: 'North America', - }, - { - name: 'country_name', - level: 'core', - type: 'keyword', - description: 'Country name.', - example: 'Canada', - }, - { - name: 'region_name', - level: 'core', - type: 'keyword', - description: 'Region name.', - example: 'Quebec', - }, - { - name: 'city_name', - level: 'core', - type: 'keyword', - description: 'City name.', - example: 'Montreal', - }, - { - name: 'country_iso_code', - level: 'core', - type: 'keyword', - description: 'Country ISO code.', - example: 'CA', - }, - { - name: 'region_iso_code', - level: 'core', - type: 'keyword', - description: 'Region ISO code.', - example: 'CA-QC', - }, - { - name: 'name', - level: 'extended', - type: 'keyword', - description: - 'User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation.', - example: 'boston-dc', - }, - ], + ignore_above: 1024, + description: 'Internal product name of the file, provided at compile-time.', + example: 'Microsoft® Windows® Operating System', + default_field: false, }, ], }, { - name: 'organization', - title: 'Organization', + name: 'dns', + title: 'DNS', group: 2, description: - 'The organization fields enrich data with information about the company or entity the data is associated with. These fields help you arrange or filter data stored in an index by one or multiple organizations.', + 'Fields describing DNS queries and answers.\n\nDNS events should either represent a single DNS query prior to getting answers\n(`dns.type:query`) or they should represent a full exchange and contain the\nquery details as well as all of the answers that were provided for this query\n(`dns.type:answer`).', type: 'group', fields: [ { - name: 'name', + name: 'answers', level: 'extended', - type: 'keyword', - description: 'Organization name.', + type: 'object', + object_type: 'keyword', + description: + 'An array containing an object for each answer section returned\nby the server.\n\nThe main keys that should be present in these objects are defined by ECS.\nRecords that have more information may contain more keys than what ECS defines.\n\nNot all DNS data sources give all details about DNS answers. At minimum, answer\nobjects must contain the `data` key. If more information is available, map\nas much of it to ECS as possible, and add any additional fields to the answer\nobjects as custom fields.', }, { - name: 'id', + name: 'answers.class', level: 'extended', type: 'keyword', - description: 'Unique identifier for the organization.', + ignore_above: 1024, + description: 'The class of DNS data contained in this resource record.', + example: 'IN', }, - ], - }, - { - name: 'os', - title: 'Operating System', - group: 2, - description: 'The OS fields contain information about the operating system.', - reusable: { - top_level: false, - expected: ['observer', 'host', 'user_agent'], - }, - type: 'group', - fields: [ { - name: 'platform', + name: 'answers.data', level: 'extended', type: 'keyword', - description: 'Operating system platform (such centos, ubuntu, windows).', - example: 'darwin', + ignore_above: 1024, + description: + 'The data describing the resource.\n\nThe meaning of this data depends on the type and class of the resource record.', + example: '10.10.10.10', }, { - name: 'name', + name: 'answers.name', level: 'extended', type: 'keyword', - example: 'Mac OS X', - description: 'Operating system name, without the version.', + ignore_above: 1024, + description: + 'The domain name to which this resource record pertains.\n\nIf a chain of CNAME is being resolved, each answer `name` should be the\none that corresponds with the answer `data`. It should not simply be the\noriginal `question.name` repeated.', + example: 'www.google.com', }, { - name: 'full', + name: 'answers.ttl', level: 'extended', - type: 'keyword', - example: 'Mac OS Mojave', - description: 'Operating system name, including the version or code name.', + type: 'long', + description: + 'The time interval in seconds that this resource record may be cached\nbefore it should be discarded. Zero values mean that the data should not be\ncached.', + example: 180, }, { - name: 'family', + name: 'answers.type', level: 'extended', type: 'keyword', - example: 'debian', - description: 'OS family (such as redhat, debian, freebsd, windows).', + ignore_above: 1024, + description: 'The type of data contained in this resource record.', + example: 'CNAME', }, { - name: 'version', + name: 'header_flags', level: 'extended', type: 'keyword', - example: '10.14.1', - description: 'Operating system version as a raw string.', + ignore_above: 1024, + description: + 'Array of 2 letter DNS header flags.\n\nExpected values are: AA, TC, RD, RA, AD, CD, DO.', + example: ['RD', 'RA'], }, { - name: 'kernel', + name: 'id', level: 'extended', type: 'keyword', - example: '4.4.0-112-generic', - description: 'Operating system kernel version as a raw string.', + ignore_above: 1024, + description: + 'The DNS packet identifier assigned by the program that generated\nthe query. The identifier is copied to the response.', + example: 62111, }, - ], - }, - { - name: 'process', - title: 'Process', - group: 2, - description: - 'These fields contain information about a process. These fields can help you correlate metrics information with a process id/name from a log message. The `process.pid` often stays in the metric itself and is copied to the global field for correlation.', - type: 'group', - fields: [ { - name: 'pid', - level: 'core', - type: 'long', - description: 'Process id.', - example: 'ssh', + name: 'op_code', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The DNS operation code that specifies the kind of query in the\nmessage. This value is set by the originator of a query and copied into the\nresponse.', + example: 'QUERY', }, { - name: 'name', + name: 'question.class', level: 'extended', type: 'keyword', - description: 'Process name. Sometimes called program name or similar.', - example: 'ssh', + ignore_above: 1024, + description: 'The class of records being queried.', + example: 'IN', }, { - name: 'ppid', + name: 'question.name', level: 'extended', - type: 'long', - description: 'Process parent id.', + type: 'keyword', + ignore_above: 1024, + description: + 'The name being queried.\n\nIf the name field contains non-printable characters (below 32 or above 126),\nthose characters should be represented as escaped base 10 integers (\\DDD).\nBack slashes and quotes should be escaped. Tabs, carriage returns, and line\nfeeds should be converted to \\t, \\r, and \\n respectively.', + example: 'www.google.com', }, { - name: 'args', + name: 'question.registered_domain', level: 'extended', type: 'keyword', - description: 'Process arguments. May be filtered to protect sensitive information.', - example: ['ssh', '-l', 'user', '10.0.0.16'], + ignore_above: 1024, + description: + 'The highest registered domain, stripped of the subdomain.\n\nFor example, the registered domain for "foo.google.com" is "google.com".\n\nThis value can be determined precisely with a list like the public suffix\nlist (http://publicsuffix.org). Trying to approximate this by simply taking\nthe last two labels will not work well for TLDs such as "co.uk".', + example: 'google.com', }, { - name: 'executable', + name: 'question.subdomain', level: 'extended', type: 'keyword', - description: 'Absolute path to the process executable.', - example: '/usr/bin/ssh', + ignore_above: 1024, + description: + 'The subdomain is all of the labels under the registered_domain.\n\nIf the domain has multiple levels of subdomain, such as "sub2.sub1.example.com",\nthe subdomain field should contain "sub2.sub1", with no trailing period.', + example: 'www', }, { - name: 'title', + name: 'question.top_level_domain', level: 'extended', type: 'keyword', + ignore_above: 1024, description: - 'Process title. The proctitle, some times the same as process name. Can also be different: for example a browser setting its title to the web page currently opened.', + 'The effective top level domain (eTLD), also known as the domain\nsuffix, is the last part of the domain name. For example, the top level domain\nfor google.com is "com".\n\nThis value can be determined precisely with a list like the public suffix\nlist (http://publicsuffix.org). Trying to approximate this by simply taking\nthe last label will not work well for effective TLDs such as "co.uk".', + example: 'co.uk', }, { - name: 'thread.id', + name: 'question.type', level: 'extended', - type: 'long', - example: 4242, - description: 'Thread ID.', + type: 'keyword', + ignore_above: 1024, + description: 'The type of record being queried.', + example: 'AAAA', }, { - name: 'start', + name: 'resolved_ip', level: 'extended', - type: 'date', - example: '2016-05-23T08:05:34.853Z', - description: 'The time the process started.', + type: 'ip', + description: + 'Array containing all IPs seen in `answers.data`.\n\nThe `answers` array can be difficult to use, because of the variety of data\nformats it can contain. Extracting all IP addresses seen in there to `dns.resolved_ip`\nmakes it possible to index them as IP addresses, and makes them easier to\nvisualize and query for.', + example: ['10.10.10.10', '10.10.10.11'], }, { - name: 'working_directory', + name: 'response_code', level: 'extended', type: 'keyword', - example: '/home/alice', - description: 'The working directory of the process.', + ignore_above: 1024, + description: 'The DNS response code.', + example: 'NOERROR', + }, + { + name: 'type', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The type of DNS event captured, query or answer.\n\nIf your source of DNS events only gives you DNS queries, you should only create\ndns events of type `dns.type:query`.\n\nIf your source of DNS events gives you answers as well, you should create\none event per query (optionally as soon as the query is seen). And a second\nevent containing all query details as well as an array of answers.', + example: 'answer', }, ], }, { - name: 'related', - title: 'Related', + name: 'ecs', + title: 'ECS', group: 2, - description: - 'This field set is meant to facilitate pivoting around a piece of data. Some pieces of information can be seen in many places in ECS. To facilitate searching for them, append values to their corresponding field in `related.`. A concrete example is IP addresses, which can be under host, observer, source, destination, client, server, and network.forwarded_ip. If you append all IPs to `related.ip`, you can then search for a given IP trivially, no matter where it appeared, by querying `related.ip:a.b.c.d`.', + description: 'Meta-information specific to ECS.', type: 'group', fields: [ { - name: 'ip', - level: 'extended', - type: 'ip', - description: 'All of the IPs seen on your event.', + name: 'version', + level: 'core', + required: true, + type: 'keyword', + ignore_above: 1024, + description: + 'ECS version this event conforms to. `ecs.version` is a required\nfield and must exist in all events.\n\nWhen querying across multiple indices -- which may conform to slightly different\nECS versions -- this field lets integrations adjust to the schema version\nof the events.', + example: '1.0.0', }, ], }, { - name: 'server', - title: 'Server', + name: 'error', + title: 'Error', group: 2, description: - 'A Server is defined as the responder in a network connection for events regarding sessions, connections, or bidirectional flow records. For TCP events, the server is the receiver of the initial SYN packet(s) of the TCP connection. For other protocols, the server is generally the responder in the network transaction. Some systems actually use the term "responder" to refer the server in TCP connections. The server fields describe details about the system acting as the server in the network event. Server fields are usually populated in conjunction with client fields. Server fields are generally not populated for packet-level events. Client / server representations can add semantic context to an exchange, which is helpful to visualize the data in certain situations. If your context falls in that category, you should still ensure that source and destination are filled appropriately.', + 'These fields can represent errors of any kind.\n\nUse them for errors that happen while fetching events or in cases where the\nevent itself contains an error.', type: 'group', fields: [ { - name: 'address', - level: 'extended', + name: 'code', + level: 'core', type: 'keyword', - description: - 'Some event server addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is.', + ignore_above: 1024, + description: 'Error code describing the error.', }, { - name: 'ip', + name: 'id', level: 'core', - type: 'ip', - description: 'IP address of the server. Can be one or multiple IPv4 or IPv6 addresses.', + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifier for the error.', }, { - name: 'port', + name: 'message', level: 'core', - type: 'long', - description: 'Port of the server.', - }, - { - name: 'mac', - level: 'core', - type: 'keyword', - description: 'MAC address of the server.', + type: 'text', + description: 'Error message.', }, { - name: 'domain', - level: 'core', + name: 'stack_trace', + level: 'extended', type: 'keyword', - description: 'Server domain.', - }, - { - name: 'bytes', - level: 'core', - type: 'long', - format: 'bytes', - example: 184, - description: 'Bytes sent from the server to the client.', - }, - { - name: 'packets', - level: 'core', - type: 'long', - example: 12, - description: 'Packets sent from the server to the client.', - }, - { - name: 'geo', - title: 'Geo', - group: 2, - description: - 'Geo fields can carry data about a specific location related to an event or geo information derived from an IP field.', - type: 'group', - fields: [ - { - name: 'location', - level: 'core', - type: 'geo_point', - description: 'Longitude and latitude.', - example: '{ "lon": -73.614830, "lat": 45.505918 }', - }, - { - name: 'continent_name', - level: 'core', - type: 'keyword', - description: 'Name of the continent.', - example: 'North America', - }, - { - name: 'country_name', - level: 'core', - type: 'keyword', - description: 'Country name.', - example: 'Canada', - }, - { - name: 'region_name', - level: 'core', - type: 'keyword', - description: 'Region name.', - example: 'Quebec', - }, - { - name: 'city_name', - level: 'core', - type: 'keyword', - description: 'City name.', - example: 'Montreal', - }, - { - name: 'country_iso_code', - level: 'core', - type: 'keyword', - description: 'Country ISO code.', - example: 'CA', - }, - { - name: 'region_iso_code', - level: 'core', - type: 'keyword', - description: 'Region ISO code.', - example: 'CA-QC', - }, + ignore_above: 1024, + multi_fields: [ { - name: 'name', - level: 'extended', - type: 'keyword', - description: - 'User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation.', - example: 'boston-dc', + name: 'text', + type: 'text', + norms: false, + default_field: false, }, ], + description: 'The stack trace of this error in plain text.', + }, + { + name: 'type', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'The type of the error, for example the class name of the exception.', + example: 'java.lang.NullPointerException', }, ], }, { - name: 'service', - title: 'Service', + name: 'event', + title: 'Event', group: 2, description: - 'The service fields describe the service for or from which the data was collected. These fields help you find and correlate logs for a specific service and version.', + 'The event fields are used for context information about the log\nor metric event itself.\n\nA log is defined as an event containing details of something that happened.\nLog events must include the time at which the thing happened. Examples of log\nevents include a process starting on a host, a network packet being sent from\na source to a destination, or a network connection between a client and a server\nbeing initiated or closed. A metric is defined as an event containing one or\nmore numerical measurements and the time at which the measurement was taken.\nExamples of metric events include memory pressure measured on a host and device\ntemperature. See the `event.kind` definition in this section for additional\ndetails about metric and state events.', type: 'group', fields: [ { - name: 'id', + name: 'action', level: 'core', type: 'keyword', + ignore_above: 1024, description: - 'Unique identifier of the running service. This id should uniquely identify this service. This makes it possible to correlate logs and metrics for one specific service. Example: If you are experiencing issues with one redis instance, you can filter on that id to see metrics and logs for that single instance.', - example: 'd37e5ebfe0ae6c4972dbe9f0174a1637bb8247f6', + 'The action captured by the event.\n\nThis describes the information in the event. It is more specific than `event.category`.\nExamples are `group-add`, `process-started`, `file-created`. The value is\nnormally defined by the implementer.', + example: 'user-password-change', }, { - name: 'name', + name: 'category', level: 'core', type: 'keyword', - example: 'elasticsearch-metrics', + ignore_above: 1024, description: - 'Name of the service data is collected from. The name of the service is normally user given. This allows if two instances of the same service are running on the same machine they can be differentiated by the `service.name`. Also it allows for distributed services that run on multiple hosts to correlate the related instances based on the name. In the case of Elasticsearch the service.name could contain the cluster name. For Beats the service.name is by default a copy of the `service.type` field if no name is specified.', + 'This is one of four ECS Categorization Fields, and indicates the\nsecond level in the ECS category hierarchy.\n\n`event.category` represents the "big buckets" of ECS categories. For example,\nfiltering on `event.category:process` yields all events relating to process\nactivity. This field is closely related to `event.type`, which is used as\na subcategory.\n\nThis field is an array. This will allow proper categorization of some events\nthat fall in multiple categories.', + example: 'authentication', }, { - name: 'type', - level: 'core', + name: 'code', + level: 'extended', type: 'keyword', - example: 'elasticsearch', + ignore_above: 1024, description: - 'The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`.', + 'Identification code for this event, if one exists.\n\nSome event sources use event codes to identify messages unambiguously, regardless\nof message language or wording adjustments over time. An example of this is\nthe Windows Event ID.', + example: 4648, }, { - name: 'state', + name: 'created', level: 'core', - type: 'keyword', - description: 'Current state of the service.', + type: 'date', + description: + 'event.created contains the date/time when the event was first\nread by an agent, or by your pipeline.\n\nThis field is distinct from @timestamp in that @timestamp typically contain\nthe time extracted from the original event.\n\nIn most situations, these two timestamps will be slightly different. The difference\ncan be used to calculate the delay between your source generating an event,\nand the time when your agent first processed it. This can be used to monitor\nyour agent or pipeline ability to keep up with your event source.\n\nIn case the two timestamps are identical, @timestamp should be used.', + example: '2016-05-23T08:05:34.857Z', }, { - name: 'version', + name: 'dataset', level: 'core', type: 'keyword', - example: '3.2.4', + ignore_above: 1024, description: - 'Version of the service the data was collected from. This allows to look at a data set only for a specific version of a service.', + 'Name of the dataset.\n\nIf an event source publishes more than one type of log or events (e.g. access\nlog, error log), the dataset is used to specify which one the event comes\nfrom.\n\nIt is recommended but not required to start the dataset name with the module\nname, followed by a dot, then the dataset name.', + example: 'apache.access', }, { - name: 'ephemeral_id', + name: 'duration', + level: 'core', + type: 'long', + format: 'duration', + input_format: 'nanoseconds', + output_format: 'asMilliseconds', + output_precision: 1, + description: + 'Duration of the event in nanoseconds.\n\nIf event.start and event.end are known this value should be the difference\nbetween the end and start time.', + }, + { + name: 'end', level: 'extended', - type: 'keyword', + type: 'date', description: - 'Ephemeral identifier of this service (if one exists). This id normally changes across restarts, but `service.id` does not.', - example: '8a4f500f', + 'event.end contains the date when the event ended or when the activity\nwas last observed.', }, - ], - }, - { - name: 'source', - title: 'Source', - group: 2, - description: - 'Source fields describe details about the source of a packet/event. Source fields are usually populated in conjunction with destination fields.', - type: 'group', - fields: [ { - name: 'address', + name: 'hash', level: 'extended', type: 'keyword', + ignore_above: 1024, description: - 'Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is.', + 'Hash (perhaps logstash fingerprint) of raw field to be able to\ndemonstrate log integrity.', + example: '123456789012345678901234567890ABCD', }, { - name: 'ip', + name: 'id', level: 'core', - type: 'ip', - description: 'IP address of the source. Can be one or multiple IPv4 or IPv6 addresses.', + type: 'keyword', + ignore_above: 1024, + description: 'Unique ID to describe the event.', + example: '8a4f500d', }, { - name: 'port', + name: 'ingested', level: 'core', - type: 'long', - description: 'Port of the source.', + type: 'date', + description: + 'Timestamp when an event arrived in the central data store.\n\nThis is different from `@timestamp`, which is when the event originally occurred. It is\nalso different from `event.created`, which is meant to capture the first time\nan agent saw the event.\n\nIn normal conditions, assuming no tampering, the timestamps should chronologically\nlook like this: `@timestamp` < `event.created` < `event.ingested`.', + example: '2016-05-23T08:05:35.101Z', + default_field: false, }, { - name: 'mac', + name: 'kind', level: 'core', type: 'keyword', - description: 'MAC address of the source.', + ignore_above: 1024, + description: + 'This is one of four ECS Categorization Fields, and indicates the\nhighest level in the ECS category hierarchy.\n\n`event.kind` gives high-level information about what type of information the\nevent contains, without being specific to the contents of the event. For example,\nvalues of this field distinguish alert events from metric events.\n\nThe value of this field can be used to inform how these kinds of events should\nbe handled. They may warrant different retention, different access control,\nit may also help understand whether the data coming in at a regular interval\nor not.', + example: 'alert', }, { - name: 'domain', + name: 'module', level: 'core', type: 'keyword', - description: 'Source domain.', + ignore_above: 1024, + description: + 'Name of the module this data is coming from.\n\nIf your monitoring agent supports the concept of modules or plugins to process\nevents of a given source (e.g. Apache logs), `event.module` should contain\nthe name of this module.', + example: 'apache', }, { - name: 'bytes', + name: 'original', level: 'core', - type: 'long', - format: 'bytes', - example: 184, - description: 'Bytes sent from the source to the destination.', + type: 'keyword', + ignore_above: 1024, + description: + 'Raw text message of entire event. Used to demonstrate log integrity.\n\nThis field is not indexed and doc_values are disabled. It cannot be searched,\nbut it can be retrieved from `_source`.', + example: + 'Sep 19 08:26:10 host CEF:0|Security| threatmanager|1.0|100|\nworm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2spt=1232', }, { - name: 'packets', + name: 'outcome', level: 'core', - type: 'long', - example: 12, - description: 'Packets sent from the source to the destination.', - }, - { - name: 'geo', - title: 'Geo', - group: 2, + type: 'keyword', + ignore_above: 1024, description: - 'Geo fields can carry data about a specific location related to an event or geo information derived from an IP field.', - type: 'group', - fields: [ - { - name: 'location', - level: 'core', - type: 'geo_point', - description: 'Longitude and latitude.', - example: '{ "lon": -73.614830, "lat": 45.505918 }', - }, - { - name: 'continent_name', - level: 'core', - type: 'keyword', - description: 'Name of the continent.', - example: 'North America', - }, - { - name: 'country_name', - level: 'core', - type: 'keyword', - description: 'Country name.', - example: 'Canada', - }, - { - name: 'region_name', - level: 'core', - type: 'keyword', - description: 'Region name.', - example: 'Quebec', - }, - { - name: 'city_name', - level: 'core', - type: 'keyword', - description: 'City name.', - example: 'Montreal', - }, - { - name: 'country_iso_code', - level: 'core', - type: 'keyword', - description: 'Country ISO code.', - example: 'CA', - }, - { - name: 'region_iso_code', - level: 'core', - type: 'keyword', - description: 'Region ISO code.', - example: 'CA-QC', - }, - { - name: 'name', - level: 'extended', - type: 'keyword', - description: - 'User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation.', - example: 'boston-dc', - }, - ], + 'This is one of four ECS Categorization Fields, and indicates the\nlowest level in the ECS category hierarchy.\n\n`event.outcome` simply denotes whether the event represents a success or a\nfailure from the perspective of the entity that produced the event.\n\nNote that when a single transaction is described in multiple events, each\nevent may populate different values of `event.outcome`, according to their\nperspective.\n\nAlso note that in the case of a compound event (a single event that contains\nmultiple logical events), this field should be populated with the value that\nbest captures the overall success or failure from the perspective of the event\nproducer.\n\nFurther note that not all events will have an associated outcome. For example,\nthis field is generally not populated for metric events, events with `event.type:info`,\nor any events for which an outcome does not make logical sense.', + example: 'success', }, - ], - }, - { - name: 'url', - title: 'URL', - description: 'URL fields provide a complete URL, with scheme, host, and path.', - type: 'group', - fields: [ { - name: 'original', + name: 'provider', level: 'extended', type: 'keyword', + ignore_above: 1024, description: - 'Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not.', - example: - 'https://www.elastic.co:443/search?q=elasticsearch#top or /search?q=elasticsearch', + 'Source of the event.\n\nEvent transports such as Syslog or the Windows Event Log typically mention\nthe source of an event. It can be the name of the software that generated\nthe event (e.g. Sysmon, httpd), or of a subsystem of the operating system\n(kernel, Microsoft-Windows-Security-Auditing).', + example: 'kernel', }, { - name: 'full', + name: 'reference', level: 'extended', type: 'keyword', + ignore_above: 1024, description: - 'If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source.', - example: 'https://www.elastic.co:443/search?q=elasticsearch#top', + 'Reference URL linking to additional information about this event.\n\nThis URL links to a static definition of the this event. Alert events, indicated\nby `event.kind:alert`, are a common use case for this field.', + example: 'https://system.vendor.com/event/#0001234', + default_field: false, }, { - name: 'scheme', - level: 'extended', - type: 'keyword', + name: 'risk_score', + level: 'core', + type: 'float', description: - 'Scheme of the request, such as "https". Note: The `:` is not part of the scheme.', - example: 'https', + "Risk score or priority of the event (e.g. security solutions).\nUse your system's original value here.", }, { - name: 'domain', + name: 'risk_score_norm', level: 'extended', - type: 'keyword', + type: 'float', description: - 'Domain of the request, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field.', - example: 'www.elastic.co', + 'Normalized risk score or priority of the event, on a scale of\n0 to 100.\n\nThis is mainly useful if you use more than one system that assigns risk scores,\nand you want to see a normalized value across all systems.', }, { - name: 'port', + name: 'sequence', level: 'extended', - type: 'integer', - description: 'Port of the request, such as 443.', - example: 443, + type: 'long', + format: 'string', + description: + 'Sequence number of the event.\n\nThe sequence number is a value published by some event sources, to make the\nexact ordering of events unambiguous, regardless of the timestamp precision.', }, { - name: 'path', - level: 'extended', - type: 'keyword', - description: 'Path of the request, such as "/search".', + name: 'severity', + level: 'core', + type: 'long', + format: 'string', + description: + 'The numeric severity of the event according to your event source.\n\nWhat the different severity values mean can be different between sources and\nuse cases. It is up to the implementer to make sure severities are consistent\nacross events from the same source.\n\nThe Syslog severity belongs in `log.syslog.severity.code`. `event.severity`\nis meant to represent the severity according to the event source (e.g. firewall,\nIDS). If the event source does not publish its own severity, you may optionally\ncopy the `log.syslog.severity.code` to `event.severity`.', + example: 7, }, { - name: 'query', + name: 'start', level: 'extended', - type: 'keyword', + type: 'date', description: - 'The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases.', + 'event.start contains the date when the event started or when the\nactivity was first observed.', }, { - name: 'fragment', + name: 'timezone', level: 'extended', type: 'keyword', + ignore_above: 1024, description: - 'Portion of the url after the `#`, such as "top". The `#` is not part of the fragment.', + 'This field should be populated when the event timestamp does\nnot include timezone information already (e.g. default Syslog timestamps).\nIt is optional otherwise.\n\nAcceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"),\nabbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00").', }, { - name: 'username', - level: 'extended', + name: 'type', + level: 'core', type: 'keyword', - description: 'Username of the request.', + ignore_above: 1024, + description: + 'This is one of four ECS Categorization Fields, and indicates the\nthird level in the ECS category hierarchy.\n\n`event.type` represents a categorization "sub-bucket" that, when used along\nwith the `event.category` field values, enables filtering events down to a\nlevel appropriate for single visualization.\n\nThis field is an array. This will allow proper categorization of some events\nthat fall in multiple event types.', }, { - name: 'password', + name: 'url', level: 'extended', type: 'keyword', - description: 'Password of the request.', + ignore_above: 1024, + description: + 'URL linking to an external system to continue investigation of\nthis event.\n\nThis URL links to another system where in-depth investigation of the specific\noccurence of this event can take place. Alert events, indicated by `event.kind:alert`,\nare a common use case for this field.', + example: 'https://mysystem.mydomain.com/alert/5271dedb-f5b0-4218-87f0-4ac4870a38fe', + default_field: false, }, ], }, { - name: 'user', - title: 'User', + name: 'file', + title: 'File', group: 2, description: - 'The user fields describe information about the user that is relevant to the event. Fields can have one entry or multiple entries. If a user has more than one id, provide an array that includes all of them.', - reusable: { - top_level: true, - expected: ['client', 'destination', 'host', 'server', 'source'], - }, + 'A file is defined as a set of information that has been created\non, or has existed on a filesystem.\n\nFile objects can be associated with host events, network events, and/or file\nevents (e.g., those produced by File Integrity Monitoring [FIM] products or\nservices). File fields provide details about the affected file associated with\nthe event or metric.', type: 'group', fields: [ { - name: 'id', - level: 'core', + name: 'accessed', + level: 'extended', + type: 'date', + description: + 'Last time the file was accessed.\n\nNote that not all filesystems keep track of access time.', + }, + { + name: 'attributes', + level: 'extended', type: 'keyword', - description: 'One or multiple unique identifiers of the user.', + ignore_above: 1024, + description: + 'Array of file attributes.\n\nAttributes names will vary by platform. Here is a non-exhaustive list of values\nthat are expected in this field: archive, compressed, directory, encrypted,\nexecute, hidden, read, readonly, system, write.', + example: '["readonly", "system"]', + default_field: false, }, { - name: 'name', + name: 'code_signature.exists', level: 'core', - type: 'keyword', - example: 'albert', - description: 'Short name or login of the user.', + type: 'boolean', + description: 'Boolean to capture if a signature is present.', + example: 'true', + default_field: false, }, { - name: 'full_name', + name: 'code_signature.status', level: 'extended', type: 'keyword', - example: 'Albert Einstein', - description: "User's full name, if available. ", + ignore_above: 1024, + description: + 'Additional information about the certificate status.\n\nThis is useful for logging cryptographic errors with the certificate validity\nor trust status. Leave unpopulated if the validity or trust of the certificate\nwas unchecked.', + example: 'ERROR_UNTRUSTED_ROOT', + default_field: false, }, { - name: 'email', - level: 'extended', + name: 'code_signature.subject_name', + level: 'core', type: 'keyword', - description: 'User email address.', + ignore_above: 1024, + description: 'Subject name of the code signer', + example: 'Microsoft Corporation', + default_field: false, }, { - name: 'hash', + name: 'code_signature.trusted', level: 'extended', - type: 'keyword', + type: 'boolean', description: - 'Unique user hash to correlate information for a user in anonymized form. Useful if `user.id` or `user.name` contain confidential information and cannot be used.', + 'Stores the trust status of the certificate chain.\n\nValidating the trust of the certificate chain may be complicated, and this\nfield should only be populated by tools that actively check the status.', + example: 'true', + default_field: false, }, { - name: 'group', - title: 'Group', - group: 2, + name: 'code_signature.valid', + level: 'extended', + type: 'boolean', description: - 'The group fields are meant to represent groups that are relevant to the event.', - type: 'group', - fields: [ - { - name: 'id', - level: 'extended', - type: 'keyword', - description: 'Unique identifier for the group on the system/platform.', - }, - { - name: 'name', - level: 'extended', - type: 'keyword', - description: 'Name of the group.', - }, - ], + 'Boolean to capture if the digital signature is verified against\nthe binary content.\n\nLeave unpopulated if a certificate was unchecked.', + example: 'true', + default_field: false, }, - ], - }, - { - name: 'user_agent', - title: 'User agent', - group: 2, - description: - 'The user_agent fields normally come from a browser request. They often show up in web service logs coming from the parsed user agent string.', - type: 'group', - fields: [ { - name: 'original', + name: 'created', level: 'extended', - type: 'keyword', - description: 'Unparsed version of the user_agent.', - example: - 'Mozilla/5.0 (iPhone; CPU iPhone OS 12_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1', + type: 'date', + description: + 'File creation time.\n\nNote that not all filesystems store the creation time.', }, { - name: 'name', + name: 'ctime', level: 'extended', - type: 'keyword', - example: 'Safari', - description: 'Name of the user agent.', + type: 'date', + description: + 'Last time the file attributes or metadata changed.\n\nNote that changes to the file content will update `mtime`. This implies `ctime`\nwill be adjusted at the same time, since `mtime` is an attribute of the file.', }, { - name: 'version', + name: 'device', level: 'extended', type: 'keyword', - description: 'Version of the user agent.', - example: 12, + ignore_above: 1024, + description: 'Device that is the source of the file.', + example: 'sda', }, { - name: 'device.name', + name: 'directory', level: 'extended', type: 'keyword', - example: 'iPhone', - description: 'Name of the device.', + ignore_above: 1024, + description: + 'Directory where the file is located. It should include the drive\nletter, when appropriate.', + example: '/home/alice', }, { - name: 'os', - title: 'Operating System', - group: 2, - description: 'The OS fields contain information about the operating system.', - reusable: { - top_level: false, - expected: ['observer', 'host', 'user_agent'], - }, - type: 'group', - fields: [ - { - name: 'platform', - level: 'extended', - type: 'keyword', - description: 'Operating system platform (such centos, ubuntu, windows).', - example: 'darwin', - }, - { - name: 'name', - level: 'extended', - type: 'keyword', - example: 'Mac OS X', - description: 'Operating system name, without the version.', - }, - { - name: 'full', - level: 'extended', - type: 'keyword', - example: 'Mac OS Mojave', - description: 'Operating system name, including the version or code name.', - }, - { - name: 'family', - level: 'extended', - type: 'keyword', - example: 'debian', - description: 'OS family (such as redhat, debian, freebsd, windows).', - }, - { - name: 'version', - level: 'extended', - type: 'keyword', - example: '10.14.1', - description: 'Operating system version as a raw string.', - }, - { - name: 'kernel', - level: 'extended', - type: 'keyword', - example: '4.4.0-112-generic', - description: 'Operating system kernel version as a raw string.', - }, - ], + name: 'drive_letter', + level: 'extended', + type: 'keyword', + ignore_above: 1, + description: + 'Drive letter where the file is located. This field is only relevant\non Windows.\n\nThe value should be uppercase, and not include the colon.', + example: 'C', + default_field: false, }, - ], - }, - { - name: 'agent.hostname', - type: 'keyword', - description: 'Hostname of the agent.', - }, - ], - }, - { - key: 'beat', - title: 'Beat', - description: 'Contains common beat fields available in all event types.', - fields: [ - { - name: 'beat.timezone', - type: 'alias', - path: 'event.timezone', - migration: true, - }, - { - name: 'fields', - type: 'object', - object_type: 'keyword', - description: 'Contains user configurable fields.', - }, - { - name: 'error', - type: 'group', - description: 'Error fields containing additional info in case of errors.', - fields: [ { - name: 'type', + name: 'extension', + level: 'extended', type: 'keyword', - description: 'Error type.', + ignore_above: 1024, + description: 'File extension.', + example: 'png', }, - ], - }, - { - name: 'beat.name', - type: 'alias', - path: 'host.name', - migration: true, - }, - { - name: 'beat.hostname', - type: 'alias', - path: 'agent.hostname', - migration: true, - }, - ], - }, - { - key: 'cloud', - title: 'Cloud provider metadata', - description: 'Metadata from cloud providers added by the add_cloud_metadata processor.', - fields: [ - { - name: 'cloud.project.id', - example: 'project-x', - description: 'Name of the project in Google Cloud.', - }, - { - name: 'meta.cloud.provider', - type: 'alias', - path: 'cloud.provider', - migration: true, - }, - { - name: 'meta.cloud.instance_id', - type: 'alias', - path: 'cloud.instance.id', - migration: true, - }, - { - name: 'meta.cloud.instance_name', - type: 'alias', - path: 'cloud.instance.name', - migration: true, - }, - { - name: 'meta.cloud.machine_type', - type: 'alias', - path: 'cloud.machine.type', - migration: true, - }, - { - name: 'meta.cloud.availability_zone', - type: 'alias', - path: 'cloud.availability_zone', - migration: true, - }, - { - name: 'meta.cloud.project_id', - type: 'alias', - path: 'cloud.project.id', - migration: true, - }, - { - name: 'meta.cloud.region', - type: 'alias', - path: 'cloud.region', - migration: true, - }, - ], - }, - { - key: 'docker', - title: 'Docker', - description: 'Docker stats collected from Docker.', - short_config: false, - anchor: 'docker-processor', - fields: [ - { - name: 'docker', - type: 'group', - fields: [ { - name: 'container.id', - type: 'alias', - path: 'container.id', - migration: true, + name: 'gid', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Primary group ID (GID) of the file.', + example: '1001', }, { - name: 'container.image', - type: 'alias', - path: 'container.image.name', - migration: true, + name: 'group', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Primary group name of the file.', + example: 'alice', }, { - name: 'container.name', - type: 'alias', - path: 'container.name', - migration: true, + name: 'hash.md5', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'MD5 hash.', }, { - name: 'container.labels', - type: 'object', - object_type: 'keyword', - description: 'Image labels.', + name: 'hash.sha1', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'SHA1 hash.', }, - ], - }, - ], - }, - { - key: 'host', - title: 'Host', - description: 'Info collected for the host machine.', - anchor: 'host-processor', - }, - { - key: 'kubernetes', - title: 'Kubernetes', - description: 'Kubernetes metadata added by the kubernetes processor', - short_config: false, - anchor: 'kubernetes-processor', - fields: [ - { - name: 'kubernetes', - type: 'group', - fields: [ { - name: 'pod.name', + name: 'hash.sha256', + level: 'extended', type: 'keyword', - description: 'Kubernetes pod name', + ignore_above: 1024, + description: 'SHA256 hash.', }, { - name: 'pod.uid', + name: 'hash.sha512', + level: 'extended', type: 'keyword', - description: 'Kubernetes Pod UID', + ignore_above: 1024, + description: 'SHA512 hash.', }, { - name: 'namespace', + name: 'inode', + level: 'extended', type: 'keyword', - description: 'Kubernetes namespace', + ignore_above: 1024, + description: 'Inode representing the file in the filesystem.', + example: '256383', }, { - name: 'node.name', + name: 'mime_type', + level: 'extended', type: 'keyword', - description: 'Kubernetes node name', + ignore_above: 1024, + description: + 'MIME type should identify the format of the file or stream of bytes\nusing https://www.iana.org/assignments/media-types/media-types.xhtml[IANA\nofficial types], where possible. When more than one type is applicable, the\nmost specific type should be used.', + default_field: false, }, { - name: 'labels', - type: 'object', - description: 'Kubernetes labels map', + name: 'mode', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Mode of the file in octal representation.', + example: '0640', }, { - name: 'annotations', - type: 'object', - description: 'Kubernetes annotations map', + name: 'mtime', + level: 'extended', + type: 'date', + description: 'Last time the file content was modified.', }, { - name: 'container.name', + name: 'name', + level: 'extended', type: 'keyword', - description: 'Kubernetes container name', + ignore_above: 1024, + description: 'Name of the file including the extension, without the directory.', + example: 'example.png', }, { - name: 'container.image', + name: 'owner', + level: 'extended', type: 'keyword', - description: 'Kubernetes container image', + ignore_above: 1024, + description: "File owner's username.", + example: 'alice', }, - ], - }, - ], - }, - { - key: 'process', - title: 'Process', - description: 'Process metadata fields', - fields: [ - { - name: 'process', - type: 'group', - fields: [ - { - name: 'exe', - type: 'alias', - path: 'process.executable', - migration: true, - }, - ], - }, - ], - }, - { - key: 'common', - title: 'Common', - description: - 'These fields contain data about the environment in which the transaction or flow was captured.', - fields: [ - { - name: 'type', - description: - 'The type of the transaction (for example, HTTP, MySQL, Redis, or RUM) or "flow" in case of flows.', - required: true, - }, - { - name: 'server.process.name', - description: 'The name of the process that served the transaction.', - }, - { - name: 'server.process.args', - description: 'The command-line of the process that served the transaction.', - }, - { - name: 'server.process.executable', - description: 'Absolute path to the server process executable.', - }, - { - name: 'server.process.working_directory', - description: 'The working directory of the server process.', - }, - { - name: 'server.process.start', - description: 'The time the server process started.', - }, - { - name: 'client.process.name', - description: 'The name of the process that initiated the transaction.', - }, - { - name: 'client.process.args', - description: 'The command-line of the process that initiated the transaction.', - }, - { - name: 'client.process.executable', - description: 'Absolute path to the client process executable.', - }, - { - name: 'client.process.working_directory', - description: 'The working directory of the client process.', - }, - { - name: 'client.process.start', - description: 'The time the client process started.', - }, - { - name: 'real_ip', - type: 'alias', - path: 'network.forwarded_ip', - migration: true, - description: - 'If the server initiating the transaction is a proxy, this field contains the original client IP address. For HTTP, for example, the IP address extracted from a configurable HTTP header, by default `X-Forwarded-For`. Unless this field is disabled, it always has a value, and it matches the `client_ip` for non proxy clients.', - }, - { - name: 'transport', - type: 'alias', - path: 'network.transport', - migration: true, - description: - 'The transport protocol used for the transaction. If not specified, then tcp is assumed.', - }, - ], - }, - { - key: 'flows_event', - title: 'Flow Event', - description: 'These fields contain data about the flow itself.', - fields: [ - { - name: 'flow.final', - type: 'boolean', - description: - 'Indicates if event is last event in flow. If final is false, the event reports an intermediate flow state only.', - }, - { - name: 'flow.id', - description: 'Internal flow ID based on connection meta data and address.', - }, - { - name: 'flow.vlan', - type: 'long', - description: - "VLAN identifier from the 802.1q frame. In case of a multi-tagged frame this field will be an array with the outer tag's VLAN identifier listed first. ", - }, - { - name: 'flow_id', - type: 'alias', - path: 'flow.id', - migration: true, - }, - { - name: 'final', - type: 'alias', - path: 'flow.final', - migration: true, - }, - { - name: 'vlan', - type: 'alias', - path: 'flow.vlan', - migration: true, - }, - { - name: 'source.stats.net_bytes_total', - type: 'alias', - path: 'source.bytes', - migration: true, - }, - { - name: 'source.stats.net_packets_total', - type: 'alias', - path: 'source.packets', - migration: true, - }, - { - name: 'dest.stats.net_bytes_total', - type: 'alias', - path: 'destination.bytes', - migration: true, - }, - { - name: 'dest.stats.net_packets_total', - type: 'alias', - path: 'destination.packets', - migration: true, - }, - ], - }, - { - key: 'trans_event', - title: 'Transaction Event', - description: 'These fields contain data about the transaction itself.', - fields: [ - { - name: 'status', - description: - 'The high level status of the transaction. The way to compute this value depends on the protocol, but the result has a meaning independent of the protocol.', - required: true, - possible_values: ['OK', 'Error', 'Server Error', 'Client Error'], - }, - { - name: 'method', - description: - 'The command/verb/method of the transaction. For HTTP, this is the method name (GET, POST, PUT, and so on), for SQL this is the verb (SELECT, UPDATE, DELETE, and so on).', - }, - { - name: 'resource', - description: - 'The logical resource that this transaction refers to. For HTTP, this is the URL path up to the last slash (/). For example, if the URL is `/users/1`, the resource is `/users`. For databases, the resource is typically the table name. The field is not filled for all transaction types.', - }, - { - name: 'path', - required: true, - description: - 'The path the transaction refers to. For HTTP, this is the URL. For SQL databases, this is the table name. For key-value stores, this is the key.', - }, - { - name: 'query', - type: 'keyword', - description: - 'The query in a human readable format. For HTTP, it will typically be something like `GET /users/_search?name=test`. For MySQL, it is something like `SELECT id from users where name=test`.', - }, - { - name: 'params', - type: 'text', - description: - 'The request parameters. For HTTP, these are the POST or GET parameters. For Thrift-RPC, these are the parameters from the request.', - }, - { - name: 'notes', - type: 'alias', - path: 'error.message', - description: - 'Messages from Packetbeat itself. This field usually contains error messages for interpreting the raw data. This information can be helpful for troubleshooting.', - }, - ], - }, - { - key: 'raw', - title: 'Raw', - description: 'These fields contain the raw transaction data.', - fields: [ - { - name: 'request', - type: 'text', - description: - 'For text protocols, this is the request as seen on the wire (application layer only). For binary protocols this is our representation of the request.', - }, - { - name: 'response', - type: 'text', - description: - 'For text protocols, this is the response as seen on the wire (application layer only). For binary protocols this is our representation of the request.', - }, - ], - }, - { - key: 'trans_measurements', - title: 'Measurements (Transactions)', - description: 'These fields contain measurements related to the transaction.', - fields: [ - { - name: 'bytes_in', - type: 'alias', - path: 'source.bytes', - description: - 'The number of bytes of the request. Note that this size is the application layer message length, without the length of the IP or TCP headers.', - }, - { - name: 'bytes_out', - type: 'alias', - path: 'destination.bytes', - description: - 'The number of bytes of the response. Note that this size is the application layer message length, without the length of the IP or TCP headers.', - }, - ], - }, - { - key: 'amqp', - title: 'AMQP', - description: 'AMQP specific event fields.', - fields: [ - { - name: 'amqp', - type: 'group', - fields: [ { - name: 'reply-code', - type: 'long', - description: 'AMQP reply code to an error, similar to http reply-code', - example: 404, + name: 'path', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: + 'Full path to the file, including the file name. It should include\nthe drive letter, when appropriate.', + example: '/home/alice/example.png', }, { - name: 'reply-text', + name: 'pe.company', + level: 'extended', type: 'keyword', - description: 'Text explaining the error.', + ignore_above: 1024, + description: 'Internal company name of the file, provided at compile-time.', + example: 'Microsoft Corporation', + default_field: false, }, { - name: 'class-id', - type: 'long', - description: 'Failing method class.', + name: 'pe.description', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal description of the file, provided at compile-time.', + example: 'Paint', + default_field: false, }, { - name: 'method-id', - type: 'long', - description: 'Failing method ID.', + name: 'pe.file_version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal version of the file, provided at compile-time.', + example: '6.3.9600.17415', + default_field: false, }, { - name: 'exchange', + name: 'pe.original_file_name', + level: 'extended', type: 'keyword', - description: 'Name of the exchange.', + ignore_above: 1024, + description: 'Internal name of the file, provided at compile-time.', + example: 'MSPAINT.EXE', + default_field: false, }, { - name: 'exchange-type', + name: 'pe.product', + level: 'extended', type: 'keyword', - description: 'Exchange type.', - example: 'fanout', + ignore_above: 1024, + description: 'Internal product name of the file, provided at compile-time.', + example: 'Microsoft® Windows® Operating System', + default_field: false, }, { - name: 'passive', - type: 'boolean', - description: 'If set, do not create exchange/queue.', + name: 'size', + level: 'extended', + type: 'long', + description: 'File size in bytes.\n\nOnly relevant when `file.type` is "file".', + example: 16384, }, { - name: 'durable', - type: 'boolean', - description: 'If set, request a durable exchange/queue.', + name: 'target_path', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Target path for symlinks.', }, { - name: 'exclusive', - type: 'boolean', - description: 'If set, request an exclusive queue.', + name: 'type', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'File type (file, dir, or symlink).', + example: 'file', }, { - name: 'auto-delete', - type: 'boolean', - description: 'If set, auto-delete queue when unused.', + name: 'uid', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'The user ID (UID) or security identifier (SID) of the file owner.', + example: '1001', }, + ], + }, + { + name: 'geo', + title: 'Geo', + group: 2, + description: + 'Geo fields can carry data about a specific location related to an\nevent.\n\nThis geolocation information can be derived from techniques such as Geo IP,\nor be user-supplied.', + type: 'group', + fields: [ { - name: 'no-wait', - type: 'boolean', - description: 'If set, the server will not respond to the method.', + name: 'city_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'City name.', + example: 'Montreal', }, { - name: 'consumer-tag', - description: 'Identifier for the consumer, valid within the current channel.', + name: 'continent_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the continent.', + example: 'North America', }, { - name: 'delivery-tag', - type: 'long', - description: 'The server-assigned and channel-specific delivery tag.', + name: 'country_iso_code', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Country ISO code.', + example: 'CA', }, { - name: 'message-count', - type: 'long', - description: - 'The number of messages in the queue, which will be zero for newly-declared queues.', + name: 'country_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Country name.', + example: 'Canada', }, { - name: 'consumer-count', - type: 'long', - description: 'The number of consumers of a queue.', + name: 'location', + level: 'core', + type: 'geo_point', + description: 'Longitude and latitude.', + example: '{ "lon": -73.614830, "lat": 45.505918 }', }, { - name: 'routing-key', + name: 'name', + level: 'extended', type: 'keyword', - description: 'Message routing key.', + ignore_above: 1024, + description: + 'User-defined description of a location, at the level of granularity\nthey care about.\n\nCould be the name of their data centers, the floor number, if this describes\na local physical entity, city names.\n\nNot typically used in automated geolocation.', + example: 'boston-dc', }, { - name: 'no-ack', - type: 'boolean', - description: 'If set, the server does not expect acknowledgements for messages.', + name: 'region_iso_code', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Region ISO code.', + example: 'CA-QC', }, { - name: 'no-local', - type: 'boolean', + name: 'region_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Region name.', + example: 'Quebec', + }, + ], + }, + { + name: 'group', + title: 'Group', + group: 2, + description: + 'The group fields are meant to represent groups that are relevant\nto the event.', + type: 'group', + fields: [ + { + name: 'domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, description: - 'If set, the server will not send messages to the connection that published them.', + 'Name of the directory the group is a member of.\n\nFor example, an LDAP or Active Directory domain name.', }, { - name: 'if-unused', - type: 'boolean', - description: 'Delete only if unused.', + name: 'id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifier for the group on the system/platform.', }, { - name: 'if-empty', - type: 'boolean', - description: 'Delete only if empty.', + name: 'name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the group.', }, + ], + }, + { + name: 'hash', + title: 'Hash', + group: 2, + description: + 'The hash fields represent different hash algorithms and their values.\n\nField names for common hashes (e.g. MD5, SHA1) are predefined. Add fields for\nother hashes by lowercasing the hash algorithm name and using underscore separators\nas appropriate (snake case, e.g. sha3_512).', + type: 'group', + fields: [ { - name: 'queue', + name: 'md5', + level: 'extended', type: 'keyword', - description: 'The queue name identifies the queue within the vhost.', + ignore_above: 1024, + description: 'MD5 hash.', }, { - name: 'redelivered', - type: 'boolean', - description: - 'Indicates that the message has been previously delivered to this or another client.', + name: 'sha1', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'SHA1 hash.', }, { - name: 'multiple', - type: 'boolean', - description: 'Acknowledge multiple messages.', + name: 'sha256', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'SHA256 hash.', }, { - name: 'arguments', - type: 'object', - description: - 'Optional additional arguments passed to some methods. Can be of various types.', + name: 'sha512', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'SHA512 hash.', }, + ], + }, + { + name: 'host', + title: 'Host', + group: 2, + description: + 'A host is defined as a general computing instance.\n\nECS host.* fields should be populated with details about the host on which the\nevent happened, or from which the measurement was taken. Host types include\nhardware, virtual machines, Docker containers, and Kubernetes nodes.', + type: 'group', + fields: [ { - name: 'mandatory', - type: 'boolean', - description: 'Indicates mandatory routing.', + name: 'architecture', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Operating system architecture.', + example: 'x86_64', }, { - name: 'immediate', - type: 'boolean', - description: 'Request immediate delivery.', + name: 'domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the domain of which the host is a member.\n\nFor example, on Windows this could be the host Active Directory domain\nor NetBIOS domain name. For Linux this could be the domain of the host\nLDAP provider.', + example: 'CONTOSO', + default_field: false, }, { - name: 'content-type', + name: 'geo.city_name', + level: 'core', type: 'keyword', - description: 'MIME content type.', - example: 'text/plain', + ignore_above: 1024, + description: 'City name.', + example: 'Montreal', }, { - name: 'content-encoding', + name: 'geo.continent_name', + level: 'core', type: 'keyword', - description: 'MIME content encoding.', + ignore_above: 1024, + description: 'Name of the continent.', + example: 'North America', }, { - name: 'headers', - type: 'object', - object_type: 'keyword', - description: 'Message header field table.', + name: 'geo.country_iso_code', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Country ISO code.', + example: 'CA', }, { - name: 'delivery-mode', + name: 'geo.country_name', + level: 'core', type: 'keyword', - description: 'Non-persistent (1) or persistent (2).', + ignore_above: 1024, + description: 'Country name.', + example: 'Canada', }, { - name: 'priority', - type: 'long', - description: 'Message priority, 0 to 9.', + name: 'geo.location', + level: 'core', + type: 'geo_point', + description: 'Longitude and latitude.', + example: '{ "lon": -73.614830, "lat": 45.505918 }', }, { - name: 'correlation-id', + name: 'geo.name', + level: 'extended', type: 'keyword', - description: 'Application correlation identifier.', + ignore_above: 1024, + description: + 'User-defined description of a location, at the level of granularity\nthey care about.\n\nCould be the name of their data centers, the floor number, if this describes\na local physical entity, city names.\n\nNot typically used in automated geolocation.', + example: 'boston-dc', }, { - name: 'reply-to', + name: 'geo.region_iso_code', + level: 'core', type: 'keyword', - description: 'Address to reply to.', + ignore_above: 1024, + description: 'Region ISO code.', + example: 'CA-QC', }, { - name: 'expiration', + name: 'geo.region_name', + level: 'core', type: 'keyword', - description: 'Message expiration specification.', + ignore_above: 1024, + description: 'Region name.', + example: 'Quebec', }, { - name: 'message-id', + name: 'hostname', + level: 'core', type: 'keyword', - description: 'Application message identifier.', + ignore_above: 1024, + description: + 'Hostname of the host.\n\nIt normally contains what the `hostname` command returns on the host machine.', }, { - name: 'timestamp', + name: 'id', + level: 'core', type: 'keyword', - description: 'Message timestamp.', + ignore_above: 1024, + description: + 'Unique host id.\n\nAs hostname is not always unique, use values that are meaningful in your environment.\n\nExample: The current usage of `beat.name`.', }, { - name: 'type', - type: 'keyword', - description: 'Message type name.', + name: 'ip', + level: 'core', + type: 'ip', + description: 'Host ip addresses.', }, { - name: 'user-id', + name: 'mac', + level: 'core', type: 'keyword', - description: 'Creating user id.', + ignore_above: 1024, + description: 'Host mac addresses.', }, { - name: 'app-id', + name: 'name', + level: 'core', type: 'keyword', - description: 'Creating application id.', + ignore_above: 1024, + description: + 'Name of the host.\n\nIt can contain what `hostname` returns on Unix systems, the fully qualified\ndomain name, or a name specified by the user. The sender decides which value\nto use.', }, - ], - }, - ], - }, - { - key: 'cassandra', - title: 'Cassandra', - description: 'Cassandra v4/3 specific event fields.', - fields: [ - { - name: 'no_request', - type: 'alias', - path: 'cassandra.no_request', - migration: true, - }, - { - name: 'cassandra', - type: 'group', - description: 'Information about the Cassandra request and response.', - fields: [ { - name: 'no_request', - type: 'boolean', - description: 'Indicates that there is no request because this is a PUSH message.', + name: 'os.family', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'OS family (such as redhat, debian, freebsd, windows).', + example: 'debian', }, { - name: 'request', - type: 'group', - description: 'Cassandra request.', - fields: [ + name: 'os.full', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ { - name: 'headers', - type: 'group', - description: 'Cassandra request headers.', - fields: [ - { - name: 'version', - type: 'long', - description: 'The version of the protocol.', - }, - { - name: 'flags', - type: 'keyword', - description: 'Flags applying to this frame.', - }, - { - name: 'stream', - type: 'keyword', - description: - 'A frame has a stream id. If a client sends a request message with the stream id X, it is guaranteed that the stream id of the response to that message will be X.', - }, - { - name: 'op', - type: 'keyword', - description: 'An operation type that distinguishes the actual message.', - }, - { - name: 'length', - type: 'long', - description: - 'A integer representing the length of the body of the frame (a frame is limited to 256MB in length).', - }, - ], + name: 'text', + type: 'text', + norms: false, + default_field: false, }, + ], + description: 'Operating system name, including the version or code name.', + example: 'Mac OS Mojave', + }, + { + name: 'os.kernel', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Operating system kernel version as a raw string.', + example: '4.4.0-112-generic', + }, + { + name: 'os.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ { - name: 'query', - type: 'keyword', - description: 'The CQL query which client send to cassandra.', + name: 'text', + type: 'text', + norms: false, + default_field: false, }, ], + description: 'Operating system name, without the version.', + example: 'Mac OS X', }, { - name: 'response', + name: 'os.platform', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Operating system platform (such centos, ubuntu, windows).', + example: 'darwin', + }, + { + name: 'os.version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Operating system version as a raw string.', + example: '10.14.1', + }, + { + name: 'type', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'Type of host.\n\nFor Cloud providers this can be the machine type like `t2.medium`. If vm,\nthis could be the container, for example, or other information meaningful\nin your environment.', + }, + { + name: 'uptime', + level: 'extended', + type: 'long', + description: 'Seconds the host has been up.', + example: 1325, + }, + { + name: 'user.domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the directory the user is a member of.\n\nFor example, an LDAP or Active Directory domain name.', + }, + { + name: 'user.email', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'User email address.', + }, + { + name: 'user.full_name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: "User's full name, if available.", + example: 'Albert Einstein', + }, + { + name: 'user.group.domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the directory the group is a member of.\n\nFor example, an LDAP or Active Directory domain name.', + }, + { + name: 'user.group.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifier for the group on the system/platform.', + }, + { + name: 'user.group.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the group.', + }, + { + name: 'user.hash', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Unique user hash to correlate information for a user in anonymized\nform.\n\nUseful if `user.id` or `user.name` contain confidential information and cannot\nbe used.', + }, + { + name: 'user.id', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifiers of the user.', + }, + { + name: 'user.name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Short name or login of the user.', + example: 'albert', + }, + ], + }, + { + name: 'http', + title: 'HTTP', + group: 2, + description: + 'Fields related to HTTP activity. Use the `url` field set to store\nthe url of the request.', + type: 'group', + fields: [ + { + name: 'request.body.bytes', + level: 'extended', + type: 'long', + format: 'bytes', + description: 'Size in bytes of the request body.', + example: 887, + }, + { + name: 'request.body.content', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'The full HTTP request body.', + example: 'Hello world', + }, + { + name: 'request.bytes', + level: 'extended', + type: 'long', + format: 'bytes', + description: 'Total size in bytes of the request (body and headers).', + example: 1437, + }, + { + name: 'request.method', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'HTTP request method.\n\nThe field value must be normalized to lowercase for querying. See the documentation\nsection "Implementing ECS".', + example: 'get, post, put', + }, + { + name: 'request.referrer', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Referrer for this HTTP request.', + example: 'https://blog.example.com/', + }, + { + name: 'response.body.bytes', + level: 'extended', + type: 'long', + format: 'bytes', + description: 'Size in bytes of the response body.', + example: 887, + }, + { + name: 'response.body.content', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'The full HTTP response body.', + example: 'Hello world', + }, + { + name: 'response.bytes', + level: 'extended', + type: 'long', + format: 'bytes', + description: 'Total size in bytes of the response (body and headers).', + example: 1437, + }, + { + name: 'response.status_code', + level: 'extended', + type: 'long', + format: 'string', + description: 'HTTP response status code.', + example: 404, + }, + { + name: 'version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'HTTP version.', + example: 1.1, + }, + ], + }, + { + name: 'interface', + title: 'Interface', + group: 2, + description: + 'The interface fields are used to record ingress and egress interface\ninformation when reported by an observer (e.g. firewall, router, load balancer)\nin the context of the observer handling a network connection. In the case of\na single observer interface (e.g. network sensor on a span port) only the observer.ingress\ninformation should be populated.', + type: 'group', + fields: [ + { + name: 'alias', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Interface alias as reported by the system, typically used in firewall\nimplementations for e.g. inside, outside, or dmz logical interface naming.', + example: 'outside', + default_field: false, + }, + { + name: 'id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Interface ID as reported by an observer (typically SNMP interface\nID).', + example: 10, + default_field: false, + }, + { + name: 'name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Interface name as reported by the system.', + example: 'eth0', + default_field: false, + }, + ], + }, + { + name: 'log', + title: 'Log', + group: 2, + description: + 'Details about the event logging mechanism or logging transport.\n\nThe log.* fields are typically populated with details about the logging mechanism\nused to create and/or transport the event. For example, syslog details belong\nunder `log.syslog.*`.\n\nThe details specific to your event source are typically not logged under `log.*`,\nbut rather in `event.*` or in other ECS fields.', + type: 'group', + fields: [ + { + name: 'level', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'Original log level of the log event.\n\nIf the source of the event provides a log level or textual severity, this\nis the one that goes in `log.level`. If your source does not specify one,\nyou may put your event transport severity here (e.g. Syslog severity).\n\nSome examples are `warn`, `err`, `i`, `informational`.', + example: 'error', + }, + { + name: 'logger', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'The name of the logger inside an application. This is usually the\nname of the class which initialized the logger, or can be a custom name.', + example: 'org.elasticsearch.bootstrap.Bootstrap', + }, + { + name: 'origin.file.line', + level: 'extended', + type: 'integer', + description: + 'The line number of the file containing the source code which originated\nthe log event.', + example: 42, + }, + { + name: 'origin.file.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The name of the file containing the source code which originated\nthe log event. Note that this is not the name of the log file.', + example: 'Bootstrap.java', + }, + { + name: 'origin.function', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'The name of the function or method which originated the log event.', + example: 'init', + }, + { + name: 'original', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'This is the original log message and contains the full log message\nbefore splitting it up in multiple parts.\n\nIn contrast to the `message` field which can contain an extracted part of\nthe log message, this field contains the original, full log message. It can\nhave already some modifications applied like encoding or new lines removed\nto clean up the log message.\n\nThis field is not indexed and doc_values are disabled so it can not be queried\nbut the value can be retrieved from `_source`.', + example: 'Sep 19 08:26:10 localhost My log', + }, + { + name: 'syslog', + level: 'extended', + type: 'object', + object_type: 'keyword', + description: + 'The Syslog metadata of the event, if the event was transmitted\nvia Syslog. Please see RFCs 5424 or 3164.', + }, + { + name: 'syslog.facility.code', + level: 'extended', + type: 'long', + format: 'string', + description: + 'The Syslog numeric facility of the log event, if available.\n\nAccording to RFCs 5424 and 3164, this value should be an integer between 0\nand 23.', + example: 23, + }, + { + name: 'syslog.facility.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'The Syslog text-based facility of the log event, if available.', + example: 'local7', + }, + { + name: 'syslog.priority', + level: 'extended', + type: 'long', + format: 'string', + description: + 'Syslog numeric priority of the event, if available.\n\nAccording to RFCs 5424 and 3164, the priority is 8 * facility + severity.\nThis number is therefore expected to contain a value between 0 and 191.', + example: 135, + }, + { + name: 'syslog.severity.code', + level: 'extended', + type: 'long', + description: + 'The Syslog numeric severity of the log event, if available.\n\nIf the event source publishing via Syslog provides a different numeric severity\nvalue (e.g. firewall, IDS), your source numeric severity should go to `event.severity`.\nIf the event source does not specify a distinct severity, you can optionally\ncopy the Syslog severity to `event.severity`.', + example: 3, + }, + { + name: 'syslog.severity.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The Syslog numeric severity of the log event, if available.\n\nIf the event source publishing via Syslog provides a different severity value\n(e.g. firewall, IDS), your source text severity should go to `log.level`.\nIf the event source does not specify a distinct severity, you can optionally\ncopy the Syslog severity to `log.level`.', + example: 'Error', + }, + ], + }, + { + name: 'network', + title: 'Network', + group: 2, + description: + 'The network is defined as the communication path over which a host\nor network event happens.\n\nThe network.* fields should be populated with details about the network activity\nassociated with an event.', + type: 'group', + fields: [ + { + name: 'application', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'A name given to an application level protocol. This can be arbitrarily\nassigned for things like microservices, but also apply to things like skype,\nicq, facebook, twitter. This would be used in situations where the vendor\nor service can be decoded such as from the source/dest IP owners, ports, or\nwire format.\n\nThe field value must be normalized to lowercase for querying. See the documentation\nsection "Implementing ECS".', + example: 'aim', + }, + { + name: 'bytes', + level: 'core', + type: 'long', + format: 'bytes', + description: + 'Total bytes transferred in both directions.\n\nIf `source.bytes` and `destination.bytes` are known, `network.bytes` is their\nsum.', + example: 368, + }, + { + name: 'community_id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'A hash of source and destination IPs and ports, as well as the\nprotocol used in a communication. This is a tool-agnostic standard to identify\nflows.\n\nLearn more at https://github.com/corelight/community-id-spec.', + example: '1:hO+sN4H+MG5MY/8hIrXPqc4ZQz0=', + }, + { + name: 'direction', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + "Direction of the network traffic.\nRecommended values are:\n * inbound\n * outbound\n * internal\n * external\n * unknown\n\nWhen mapping events from a host-based monitoring context, populate this field from the host's point of view.\nWhen mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of your network perimeter.", + example: 'inbound', + }, + { + name: 'forwarded_ip', + level: 'core', + type: 'ip', + description: 'Host IP address when the source IP address is the proxy.', + example: '192.1.1.2', + }, + { + name: 'iana_number', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'IANA Protocol Number (https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml).\nStandardized list of protocols. This aligns well with NetFlow and sFlow related\nlogs which use the IANA Protocol Number.', + example: 6, + }, + { + name: 'inner', + level: 'extended', + type: 'object', + object_type: 'keyword', + description: + 'Network.inner fields are added in addition to network.vlan fields\nto describe the innermost VLAN when q-in-q VLAN tagging is present. Allowed\nfields include vlan.id and vlan.name. Inner vlan fields are typically used\nwhen sending traffic with multiple 802.1q encapsulations to a network sensor\n(e.g. Zeek, Wireshark.)', + default_field: false, + }, + { + name: 'inner.vlan.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'VLAN ID as reported by the observer.', + example: 10, + default_field: false, + }, + { + name: 'inner.vlan.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Optional VLAN name as reported by the observer.', + example: 'outside', + default_field: false, + }, + { + name: 'name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Name given by operators to sections of their network.', + example: 'Guest Wifi', + }, + { + name: 'packets', + level: 'core', + type: 'long', + description: + 'Total packets transferred in both directions.\n\nIf `source.packets` and `destination.packets` are known, `network.packets`\nis their sum.', + example: 24, + }, + { + name: 'protocol', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'L7 Network protocol name. ex. http, lumberjack, transport protocol.\n\nThe field value must be normalized to lowercase for querying. See the documentation\nsection "Implementing ECS".', + example: 'http', + }, + { + name: 'transport', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'Same as network.iana_number, but instead using the Keyword name\nof the transport layer (udp, tcp, ipv6-icmp, etc.)\n\nThe field value must be normalized to lowercase for querying. See the documentation\nsection "Implementing ECS".', + example: 'tcp', + }, + { + name: 'type', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'In the OSI Model this would be the Network Layer. ipv4, ipv6,\nipsec, pim, etc\n\nThe field value must be normalized to lowercase for querying. See the documentation\nsection "Implementing ECS".', + example: 'ipv4', + }, + { + name: 'vlan.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'VLAN ID as reported by the observer.', + example: 10, + default_field: false, + }, + { + name: 'vlan.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Optional VLAN name as reported by the observer.', + example: 'outside', + default_field: false, + }, + ], + }, + { + name: 'observer', + title: 'Observer', + group: 2, + description: + 'An observer is defined as a special network, security, or application\ndevice used to detect, observe, or create network, security, or application-related\nevents and metrics.\n\nThis could be a custom hardware appliance or a server that has been configured\nto run special network, security, or application software. Examples include\nfirewalls, web proxies, intrusion detection/prevention systems, network monitoring\nsensors, web application firewalls, data loss prevention systems, and APM servers.\nThe observer.* fields shall be populated with details of the system, if any,\nthat detects, observes and/or creates a network, security, or application event\nor metric. Message queues and ETL components used in processing events or metrics\nare not considered observers in ECS.', + type: 'group', + fields: [ + { + name: 'egress', + level: 'extended', + type: 'object', + object_type: 'keyword', + description: + 'Observer.egress holds information like interface number and name,\nvlan, and zone information to classify egress traffic. Single armed monitoring\nsuch as a network sensor on a span port should only use observer.ingress\nto categorize traffic.', + default_field: false, + }, + { + name: 'egress.interface.alias', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Interface alias as reported by the system, typically used in firewall\nimplementations for e.g. inside, outside, or dmz logical interface naming.', + example: 'outside', + default_field: false, + }, + { + name: 'egress.interface.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Interface ID as reported by an observer (typically SNMP interface\nID).', + example: 10, + default_field: false, + }, + { + name: 'egress.interface.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Interface name as reported by the system.', + example: 'eth0', + default_field: false, + }, + { + name: 'egress.vlan.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'VLAN ID as reported by the observer.', + example: 10, + default_field: false, + }, + { + name: 'egress.vlan.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Optional VLAN name as reported by the observer.', + example: 'outside', + default_field: false, + }, + { + name: 'egress.zone', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Network zone of outbound traffic as reported by the observer to\ncategorize the destination area of egress traffic, e.g. Internal, External,\nDMZ, HR, Legal, etc.', + example: 'Public_Internet', + default_field: false, + }, + { + name: 'geo.city_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'City name.', + example: 'Montreal', + }, + { + name: 'geo.continent_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the continent.', + example: 'North America', + }, + { + name: 'geo.country_iso_code', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Country ISO code.', + example: 'CA', + }, + { + name: 'geo.country_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Country name.', + example: 'Canada', + }, + { + name: 'geo.location', + level: 'core', + type: 'geo_point', + description: 'Longitude and latitude.', + example: '{ "lon": -73.614830, "lat": 45.505918 }', + }, + { + name: 'geo.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'User-defined description of a location, at the level of granularity\nthey care about.\n\nCould be the name of their data centers, the floor number, if this describes\na local physical entity, city names.\n\nNot typically used in automated geolocation.', + example: 'boston-dc', + }, + { + name: 'geo.region_iso_code', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Region ISO code.', + example: 'CA-QC', + }, + { + name: 'geo.region_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Region name.', + example: 'Quebec', + }, + { + name: 'hostname', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Hostname of the observer.', + }, + { + name: 'ingress', + level: 'extended', + type: 'object', + object_type: 'keyword', + description: + 'Observer.ingress holds information like interface number and name,\nvlan, and zone information to classify ingress traffic. Single armed monitoring\nsuch as a network sensor on a span port should only use observer.ingress\nto categorize traffic.', + default_field: false, + }, + { + name: 'ingress.interface.alias', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Interface alias as reported by the system, typically used in firewall\nimplementations for e.g. inside, outside, or dmz logical interface naming.', + example: 'outside', + default_field: false, + }, + { + name: 'ingress.interface.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Interface ID as reported by an observer (typically SNMP interface\nID).', + example: 10, + default_field: false, + }, + { + name: 'ingress.interface.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Interface name as reported by the system.', + example: 'eth0', + default_field: false, + }, + { + name: 'ingress.vlan.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'VLAN ID as reported by the observer.', + example: 10, + default_field: false, + }, + { + name: 'ingress.vlan.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Optional VLAN name as reported by the observer.', + example: 'outside', + default_field: false, + }, + { + name: 'ingress.zone', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Network zone of incoming traffic as reported by the observer to\ncategorize the source area of ingress traffic. e.g. internal, External, DMZ,\nHR, Legal, etc.', + example: 'DMZ', + default_field: false, + }, + { + name: 'ip', + level: 'core', + type: 'ip', + description: 'IP addresses of the observer.', + }, + { + name: 'mac', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'MAC addresses of the observer', + }, + { + name: 'name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Custom name of the observer.\n\nThis is a name that can be given to an observer. This can be helpful for example\nif multiple firewalls of the same model are used in an organization.\n\nIf no custom name is needed, the field can be left empty.', + example: '1_proxySG', + }, + { + name: 'os.family', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'OS family (such as redhat, debian, freebsd, windows).', + example: 'debian', + }, + { + name: 'os.full', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Operating system name, including the version or code name.', + example: 'Mac OS Mojave', + }, + { + name: 'os.kernel', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Operating system kernel version as a raw string.', + example: '4.4.0-112-generic', + }, + { + name: 'os.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Operating system name, without the version.', + example: 'Mac OS X', + }, + { + name: 'os.platform', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Operating system platform (such centos, ubuntu, windows).', + example: 'darwin', + }, + { + name: 'os.version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Operating system version as a raw string.', + example: '10.14.1', + }, + { + name: 'product', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'The product name of the observer.', + example: 's200', + }, + { + name: 'serial_number', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Observer serial number.', + }, + { + name: 'type', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'The type of the observer the data is coming from.\n\nThere is no predefined list of observer types. Some examples are `forwarder`,\n`firewall`, `ids`, `ips`, `proxy`, `poller`, `sensor`, `APM server`.', + example: 'firewall', + }, + { + name: 'vendor', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Vendor name of the observer.', + example: 'Symantec', + }, + { + name: 'version', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Observer version.', + }, + ], + }, + { + name: 'organization', + title: 'Organization', + group: 2, + description: + 'The organization fields enrich data with information about the company\nor entity the data is associated with.\n\nThese fields help you arrange or filter data stored in an index by one or multiple\norganizations.', + type: 'group', + fields: [ + { + name: 'id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifier for the organization.', + }, + { + name: 'name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Organization name.', + }, + ], + }, + { + name: 'os', + title: 'Operating System', + group: 2, + description: 'The OS fields contain information about the operating system.', + type: 'group', + fields: [ + { + name: 'family', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'OS family (such as redhat, debian, freebsd, windows).', + example: 'debian', + }, + { + name: 'full', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Operating system name, including the version or code name.', + example: 'Mac OS Mojave', + }, + { + name: 'kernel', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Operating system kernel version as a raw string.', + example: '4.4.0-112-generic', + }, + { + name: 'name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Operating system name, without the version.', + example: 'Mac OS X', + }, + { + name: 'platform', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Operating system platform (such centos, ubuntu, windows).', + example: 'darwin', + }, + { + name: 'version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Operating system version as a raw string.', + example: '10.14.1', + }, + ], + }, + { + name: 'package', + title: 'Package', + group: 2, + description: + 'These fields contain information about an installed software package.\nIt contains general information about a package, such as name, version or size.\nIt also contains installation details, such as time or location.', + type: 'group', + fields: [ + { + name: 'architecture', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Package architecture.', + example: 'x86_64', + }, + { + name: 'build_version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Additional information about the build version of the installed\npackage.\n\nFor example use the commit SHA of a non-released package.', + example: '36f4f7e89dd61b0988b12ee000b98966867710cd', + default_field: false, + }, + { + name: 'checksum', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Checksum of the installed package for verification.', + example: '68b329da9893e34099c7d8ad5cb9c940', + }, + { + name: 'description', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Description of the package.', + example: + 'Open source programming language to build simple/reliable/efficient\nsoftware.', + }, + { + name: 'install_scope', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Indicating how the package was installed, e.g. user-local, global.', + example: 'global', + }, + { + name: 'installed', + level: 'extended', + type: 'date', + description: 'Time when package was installed.', + }, + { + name: 'license', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'License under which the package was released.\n\nUse a short name, e.g. the license identifier from SPDX License List where\npossible (https://spdx.org/licenses/).', + example: 'Apache License 2.0', + }, + { + name: 'name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Package name', + example: 'go', + }, + { + name: 'path', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Path where the package is installed.', + example: '/usr/local/Cellar/go/1.12.9/', + }, + { + name: 'reference', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Home page or reference URL of the software in this package, if\navailable.', + example: 'https://golang.org', + default_field: false, + }, + { + name: 'size', + level: 'extended', + type: 'long', + format: 'string', + description: 'Package size in bytes.', + example: 62231, + }, + { + name: 'type', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Type of package.\n\nThis should contain the package file type, rather than the package manager\nname. Examples: rpm, dpkg, brew, npm, gem, nupkg, jar.', + example: 'rpm', + default_field: false, + }, + { + name: 'version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Package version', + example: '1.12.9', + }, + ], + }, + { + name: 'pe', + title: 'PE Header', + group: 2, + description: 'These fields contain Windows Portable Executable (PE) metadata.', + type: 'group', + fields: [ + { + name: 'company', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal company name of the file, provided at compile-time.', + example: 'Microsoft Corporation', + default_field: false, + }, + { + name: 'description', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal description of the file, provided at compile-time.', + example: 'Paint', + default_field: false, + }, + { + name: 'file_version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal version of the file, provided at compile-time.', + example: '6.3.9600.17415', + default_field: false, + }, + { + name: 'original_file_name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal name of the file, provided at compile-time.', + example: 'MSPAINT.EXE', + default_field: false, + }, + { + name: 'product', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal product name of the file, provided at compile-time.', + example: 'Microsoft® Windows® Operating System', + default_field: false, + }, + ], + }, + { + name: 'process', + title: 'Process', + group: 2, + description: + 'These fields contain information about a process.\n\nThese fields can help you correlate metrics information with a process id/name\nfrom a log message. The `process.pid` often stays in the metric itself and\nis copied to the global field for correlation.', + type: 'group', + fields: [ + { + name: 'args', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Array of process arguments, starting with the absolute path to\nthe executable.\n\nMay be filtered to protect sensitive information.', + example: ['/usr/bin/ssh', '-l', 'user', '10.0.0.16'], + }, + { + name: 'args_count', + level: 'extended', + type: 'long', + description: + 'Length of the process.args array.\n\nThis field can be useful for querying or performing bucket analysis on how\nmany arguments were provided to start a process. More arguments may be an\nindication of suspicious activity.', + example: 4, + default_field: false, + }, + { + name: 'code_signature.exists', + level: 'core', + type: 'boolean', + description: 'Boolean to capture if a signature is present.', + example: 'true', + default_field: false, + }, + { + name: 'code_signature.status', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Additional information about the certificate status.\n\nThis is useful for logging cryptographic errors with the certificate validity\nor trust status. Leave unpopulated if the validity or trust of the certificate\nwas unchecked.', + example: 'ERROR_UNTRUSTED_ROOT', + default_field: false, + }, + { + name: 'code_signature.subject_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Subject name of the code signer', + example: 'Microsoft Corporation', + default_field: false, + }, + { + name: 'code_signature.trusted', + level: 'extended', + type: 'boolean', + description: + 'Stores the trust status of the certificate chain.\n\nValidating the trust of the certificate chain may be complicated, and this\nfield should only be populated by tools that actively check the status.', + example: 'true', + default_field: false, + }, + { + name: 'code_signature.valid', + level: 'extended', + type: 'boolean', + description: + 'Boolean to capture if the digital signature is verified against\nthe binary content.\n\nLeave unpopulated if a certificate was unchecked.', + example: 'true', + default_field: false, + }, + { + name: 'command_line', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + }, + ], + description: + 'Full command line that started the process, including the absolute\npath to the executable, and all arguments.\n\nSome arguments may be filtered to protect sensitive information.', + example: '/usr/bin/ssh -l user 10.0.0.16', + default_field: false, + }, + { + name: 'entity_id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Unique identifier for the process.\n\nThe implementation of this is specified by the data source, but some examples\nof what could be used here are a process-generated UUID, Sysmon Process GUIDs,\nor a hash of some uniquely identifying components of a process.\n\nConstructing a globally unique identifier is a common practice to mitigate\nPID reuse as well as to identify a specific process over time, across multiple\nmonitored hosts.', + example: 'c2c455d9f99375d', + default_field: false, + }, + { + name: 'executable', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Absolute path to the process executable.', + example: '/usr/bin/ssh', + }, + { + name: 'exit_code', + level: 'extended', + type: 'long', + description: + 'The exit code of the process, if this is a termination event.\n\nThe field should be absent if there is no exit code for the event (e.g. process\nstart).', + example: 137, + default_field: false, + }, + { + name: 'hash.md5', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'MD5 hash.', + }, + { + name: 'hash.sha1', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'SHA1 hash.', + }, + { + name: 'hash.sha256', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'SHA256 hash.', + }, + { + name: 'hash.sha512', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'SHA512 hash.', + }, + { + name: 'name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Process name.\n\nSometimes called program name or similar.', + example: 'ssh', + }, + { + name: 'parent.args', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Array of process arguments.\n\nMay be filtered to protect sensitive information.', + example: ['ssh', '-l', 'user', '10.0.0.16'], + default_field: false, + }, + { + name: 'parent.args_count', + level: 'extended', + type: 'long', + description: + 'Length of the process.args array.\n\nThis field can be useful for querying or performing bucket analysis on how\nmany arguments were provided to start a process. More arguments may be an\nindication of suspicious activity.', + example: 4, + default_field: false, + }, + { + name: 'parent.code_signature.exists', + level: 'core', + type: 'boolean', + description: 'Boolean to capture if a signature is present.', + example: 'true', + default_field: false, + }, + { + name: 'parent.code_signature.status', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Additional information about the certificate status.\n\nThis is useful for logging cryptographic errors with the certificate validity\nor trust status. Leave unpopulated if the validity or trust of the certificate\nwas unchecked.', + example: 'ERROR_UNTRUSTED_ROOT', + default_field: false, + }, + { + name: 'parent.code_signature.subject_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Subject name of the code signer', + example: 'Microsoft Corporation', + default_field: false, + }, + { + name: 'parent.code_signature.trusted', + level: 'extended', + type: 'boolean', + description: + 'Stores the trust status of the certificate chain.\n\nValidating the trust of the certificate chain may be complicated, and this\nfield should only be populated by tools that actively check the status.', + example: 'true', + default_field: false, + }, + { + name: 'parent.code_signature.valid', + level: 'extended', + type: 'boolean', + description: + 'Boolean to capture if the digital signature is verified against\nthe binary content.\n\nLeave unpopulated if a certificate was unchecked.', + example: 'true', + default_field: false, + }, + { + name: 'parent.command_line', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + }, + ], + description: + 'Full command line that started the process, including the absolute\npath to the executable, and all arguments.\n\nSome arguments may be filtered to protect sensitive information.', + example: '/usr/bin/ssh -l user 10.0.0.16', + default_field: false, + }, + { + name: 'parent.entity_id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Unique identifier for the process.\n\nThe implementation of this is specified by the data source, but some examples\nof what could be used here are a process-generated UUID, Sysmon Process GUIDs,\nor a hash of some uniquely identifying components of a process.\n\nConstructing a globally unique identifier is a common practice to mitigate\nPID reuse as well as to identify a specific process over time, across multiple\nmonitored hosts.', + example: 'c2c455d9f99375d', + default_field: false, + }, + { + name: 'parent.executable', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + }, + ], + description: 'Absolute path to the process executable.', + example: '/usr/bin/ssh', + default_field: false, + }, + { + name: 'parent.exit_code', + level: 'extended', + type: 'long', + description: + 'The exit code of the process, if this is a termination event.\n\nThe field should be absent if there is no exit code for the event (e.g. process\nstart).', + example: 137, + default_field: false, + }, + { + name: 'parent.hash.md5', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'MD5 hash.', + default_field: false, + }, + { + name: 'parent.hash.sha1', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'SHA1 hash.', + default_field: false, + }, + { + name: 'parent.hash.sha256', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'SHA256 hash.', + default_field: false, + }, + { + name: 'parent.hash.sha512', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'SHA512 hash.', + default_field: false, + }, + { + name: 'parent.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + }, + ], + description: 'Process name.\n\nSometimes called program name or similar.', + example: 'ssh', + default_field: false, + }, + { + name: 'parent.pgid', + level: 'extended', + type: 'long', + format: 'string', + description: 'Identifier of the group of processes the process belongs to.', + default_field: false, + }, + { + name: 'parent.pid', + level: 'core', + type: 'long', + format: 'string', + description: 'Process id.', + example: 4242, + default_field: false, + }, + { + name: 'parent.ppid', + level: 'extended', + type: 'long', + format: 'string', + description: "Parent process' pid.", + example: 4241, + default_field: false, + }, + { + name: 'parent.start', + level: 'extended', + type: 'date', + description: 'The time the process started.', + example: '2016-05-23T08:05:34.853Z', + default_field: false, + }, + { + name: 'parent.thread.id', + level: 'extended', + type: 'long', + format: 'string', + description: 'Thread ID.', + example: 4242, + default_field: false, + }, + { + name: 'parent.thread.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Thread name.', + example: 'thread-0', + default_field: false, + }, + { + name: 'parent.title', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + }, + ], + description: + 'Process title.\n\nThe proctitle, some times the same as process name. Can also be different:\nfor example a browser setting its title to the web page currently opened.', + default_field: false, + }, + { + name: 'parent.uptime', + level: 'extended', + type: 'long', + description: 'Seconds the process has been up.', + example: 1325, + default_field: false, + }, + { + name: 'parent.working_directory', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + }, + ], + description: 'The working directory of the process.', + example: '/home/alice', + default_field: false, + }, + { + name: 'pe.company', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal company name of the file, provided at compile-time.', + example: 'Microsoft Corporation', + default_field: false, + }, + { + name: 'pe.description', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal description of the file, provided at compile-time.', + example: 'Paint', + default_field: false, + }, + { + name: 'pe.file_version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal version of the file, provided at compile-time.', + example: '6.3.9600.17415', + default_field: false, + }, + { + name: 'pe.original_file_name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal name of the file, provided at compile-time.', + example: 'MSPAINT.EXE', + default_field: false, + }, + { + name: 'pe.product', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Internal product name of the file, provided at compile-time.', + example: 'Microsoft® Windows® Operating System', + default_field: false, + }, + { + name: 'pgid', + level: 'extended', + type: 'long', + format: 'string', + description: 'Identifier of the group of processes the process belongs to.', + }, + { + name: 'pid', + level: 'core', + type: 'long', + format: 'string', + description: 'Process id.', + example: 4242, + }, + { + name: 'ppid', + level: 'extended', + type: 'long', + format: 'string', + description: "Parent process' pid.", + example: 4241, + }, + { + name: 'start', + level: 'extended', + type: 'date', + description: 'The time the process started.', + example: '2016-05-23T08:05:34.853Z', + }, + { + name: 'thread.id', + level: 'extended', + type: 'long', + format: 'string', + description: 'Thread ID.', + example: 4242, + }, + { + name: 'thread.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Thread name.', + example: 'thread-0', + }, + { + name: 'title', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: + 'Process title.\n\nThe proctitle, some times the same as process name. Can also be different:\nfor example a browser setting its title to the web page currently opened.', + }, + { + name: 'uptime', + level: 'extended', + type: 'long', + description: 'Seconds the process has been up.', + example: 1325, + }, + { + name: 'working_directory', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'The working directory of the process.', + example: '/home/alice', + }, + ], + }, + { + name: 'registry', + title: 'Registry', + group: 2, + description: 'Fields related to Windows Registry operations.', + type: 'group', + fields: [ + { + name: 'data.bytes', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Original bytes written with base64 encoding.\n\nFor Windows registry operations, such as SetValueEx and RegQueryValueEx, this\ncorresponds to the data pointed by `lp_data`. This is optional but provides\nbetter recoverability and should be populated for REG_BINARY encoded values.', + example: 'ZQBuAC0AVQBTAAAAZQBuAAAAAAA=', + default_field: false, + }, + { + name: 'data.strings', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'Content when writing string types.\n\nPopulated as an array when writing string data to the registry. For single\nstring registry types (REG_SZ, REG_EXPAND_SZ), this should be an array with\none string. For sequences of string with REG_MULTI_SZ, this array will be\nvariable length. For numeric data, such as REG_DWORD and REG_QWORD, this should\nbe populated with the decimal representation (e.g `"1"`).', + example: '["C:\\rta\\red_ttp\\bin\\myapp.exe"]', + default_field: false, + }, + { + name: 'data.type', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Standard registry type for encoding contents', + example: 'REG_SZ', + default_field: false, + }, + { + name: 'hive', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Abbreviated name for the hive.', + example: 'HKLM', + default_field: false, + }, + { + name: 'key', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Hive-relative path of keys.', + example: + 'SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Image File Execution Options\\winword.exe', + default_field: false, + }, + { + name: 'path', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Full path, including hive, key and value', + example: + 'HKLM\\SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Image File Execution\nOptions\\winword.exe\\Debugger', + default_field: false, + }, + { + name: 'value', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the value written.', + example: 'Debugger', + default_field: false, + }, + ], + }, + { + name: 'related', + title: 'Related', + group: 2, + description: + 'This field set is meant to facilitate pivoting around a piece of\ndata.\n\nSome pieces of information can be seen in many places in an ECS event. To facilitate\nsearching for them, store an array of all seen values to their corresponding\nfield in `related.`.\n\nA concrete example is IP addresses, which can be under host, observer, source,\ndestination, client, server, and network.forwarded_ip. If you append all IPs\nto `related.ip`, you can then search for a given IP trivially, no matter where\nit appeared, by querying `related.ip:192.0.2.15`.', + type: 'group', + fields: [ + { + name: 'hash', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + "All the hashes seen on your event. Populating this field, then\nusing it to search for hashes can help in situations where you're unsure what\nthe hash algorithm is (and therefore which key name to search).", + default_field: false, + }, + { + name: 'ip', + level: 'extended', + type: 'ip', + description: 'All of the IPs seen on your event.', + }, + { + name: 'user', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'All the user names seen on your event.', + default_field: false, + }, + ], + }, + { + name: 'rule', + title: 'Rule', + group: 2, + description: + 'Rule fields are used to capture the specifics of any observer or\nagent rules that generate alerts or other notable events.\n\nExamples of data sources that would populate the rule fields include: network\nadmission control platforms, network or host IDS/IPS, network firewalls, web\napplication firewalls, url filters, endpoint detection and response (EDR) systems,\netc.', + type: 'group', + fields: [ + { + name: 'author', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name, organization, or pseudonym of the author or authors who created\nthe rule used to generate this event.', + example: ['Star-Lord'], + default_field: false, + }, + { + name: 'category', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'A categorization value keyword used by the entity using the rule\nfor detection of this event.', + example: 'Attempted Information Leak', + default_field: false, + }, + { + name: 'description', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'The description of the rule generating the event.', + example: 'Block requests to public DNS over HTTPS / TLS protocols', + default_field: false, + }, + { + name: 'id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'A rule ID that is unique within the scope of an agent, observer,\nor other entity using the rule for detection of this event.', + example: 101, + default_field: false, + }, + { + name: 'license', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the license under which the rule used to generate this\nevent is made available.', + example: 'Apache 2.0', + default_field: false, + }, + { + name: 'name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'The name of the rule or signature generating the event.', + example: 'BLOCK_DNS_over_TLS', + default_field: false, + }, + { + name: 'reference', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Reference URL to additional information about the rule used to\ngenerate this event.\n\nThe URL can point to the vendor documentation about the rule. If that is\nnot available, it can also be a link to a more general page describing this\ntype of alert.', + example: 'https://en.wikipedia.org/wiki/DNS_over_TLS', + default_field: false, + }, + { + name: 'ruleset', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the ruleset, policy, group, or parent category in which\nthe rule used to generate this event is a member.', + example: 'Standard_Protocol_Filters', + default_field: false, + }, + { + name: 'uuid', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'A rule ID that is unique within the scope of a set or group of\nagents, observers, or other entities using the rule for detection of this\nevent.', + example: 1100110011, + default_field: false, + }, + { + name: 'version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'The version / revision of the rule being used for analysis.', + example: 1.1, + default_field: false, + }, + ], + }, + { + name: 'server', + title: 'Server', + group: 2, + description: + 'A Server is defined as the responder in a network connection for\nevents regarding sessions, connections, or bidirectional flow records.\n\nFor TCP events, the server is the receiver of the initial SYN packet(s) of the\nTCP connection. For other protocols, the server is generally the responder in\nthe network transaction. Some systems actually use the term "responder" to refer\nthe server in TCP connections. The server fields describe details about the\nsystem acting as the server in the network event. Server fields are usually\npopulated in conjunction with client fields. Server fields are generally not\npopulated for packet-level events.\n\nClient / server representations can add semantic context to an exchange, which\nis helpful to visualize the data in certain situations. If your context falls\nin that category, you should still ensure that source and destination are filled\nappropriately.', + type: 'group', + fields: [ + { + name: 'address', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Some event server addresses are defined ambiguously. The event\nwill sometimes list an IP, a domain or a unix socket. You should always store\nthe raw address in the `.address` field.\n\nThen it should be duplicated to `.ip` or `.domain`, depending on which one\nit is.', + }, + { + name: 'as.number', + level: 'extended', + type: 'long', + description: + 'Unique number allocated to the autonomous system. The autonomous\nsystem number (ASN) uniquely identifies each network on the Internet.', + example: 15169, + }, + { + name: 'as.organization.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Organization name.', + example: 'Google LLC', + }, + { + name: 'bytes', + level: 'core', + type: 'long', + format: 'bytes', + description: 'Bytes sent from the server to the client.', + example: 184, + }, + { + name: 'domain', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Server domain.', + }, + { + name: 'geo.city_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'City name.', + example: 'Montreal', + }, + { + name: 'geo.continent_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the continent.', + example: 'North America', + }, + { + name: 'geo.country_iso_code', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Country ISO code.', + example: 'CA', + }, + { + name: 'geo.country_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Country name.', + example: 'Canada', + }, + { + name: 'geo.location', + level: 'core', + type: 'geo_point', + description: 'Longitude and latitude.', + example: '{ "lon": -73.614830, "lat": 45.505918 }', + }, + { + name: 'geo.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'User-defined description of a location, at the level of granularity\nthey care about.\n\nCould be the name of their data centers, the floor number, if this describes\na local physical entity, city names.\n\nNot typically used in automated geolocation.', + example: 'boston-dc', + }, + { + name: 'geo.region_iso_code', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Region ISO code.', + example: 'CA-QC', + }, + { + name: 'geo.region_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Region name.', + example: 'Quebec', + }, + { + name: 'ip', + level: 'core', + type: 'ip', + description: + 'IP address of the server.\n\nCan be one or multiple IPv4 or IPv6 addresses.', + }, + { + name: 'mac', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'MAC address of the server.', + }, + { + name: 'nat.ip', + level: 'extended', + type: 'ip', + description: + 'Translated ip of destination based NAT sessions (e.g. internet\nto private DMZ)\n\nTypically used with load balancers, firewalls, or routers.', + }, + { + name: 'nat.port', + level: 'extended', + type: 'long', + format: 'string', + description: + 'Translated port of destination based NAT sessions (e.g. internet\nto private DMZ)\n\nTypically used with load balancers, firewalls, or routers.', + }, + { + name: 'packets', + level: 'core', + type: 'long', + description: 'Packets sent from the server to the client.', + example: 12, + }, + { + name: 'port', + level: 'core', + type: 'long', + format: 'string', + description: 'Port of the server.', + }, + { + name: 'registered_domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The highest registered server domain, stripped of the subdomain.\n\nFor example, the registered domain for "foo.google.com" is "google.com".\n\nThis value can be determined precisely with a list like the public suffix\nlist (http://publicsuffix.org). Trying to approximate this by simply taking\nthe last two labels will not work well for TLDs such as "co.uk".', + example: 'google.com', + }, + { + name: 'top_level_domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The effective top level domain (eTLD), also known as the domain\nsuffix, is the last part of the domain name. For example, the top level domain\nfor google.com is "com".\n\nThis value can be determined precisely with a list like the public suffix\nlist (http://publicsuffix.org). Trying to approximate this by simply taking\nthe last label will not work well for effective TLDs such as "co.uk".', + example: 'co.uk', + }, + { + name: 'user.domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the directory the user is a member of.\n\nFor example, an LDAP or Active Directory domain name.', + }, + { + name: 'user.email', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'User email address.', + }, + { + name: 'user.full_name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: "User's full name, if available.", + example: 'Albert Einstein', + }, + { + name: 'user.group.domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the directory the group is a member of.\n\nFor example, an LDAP or Active Directory domain name.', + }, + { + name: 'user.group.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifier for the group on the system/platform.', + }, + { + name: 'user.group.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the group.', + }, + { + name: 'user.hash', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Unique user hash to correlate information for a user in anonymized\nform.\n\nUseful if `user.id` or `user.name` contain confidential information and cannot\nbe used.', + }, + { + name: 'user.id', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifiers of the user.', + }, + { + name: 'user.name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Short name or login of the user.', + example: 'albert', + }, + ], + }, + { + name: 'service', + title: 'Service', + group: 2, + description: + 'The service fields describe the service for or from which the data\nwas collected.\n\nThese fields help you find and correlate logs for a specific service and version.', + type: 'group', + fields: [ + { + name: 'ephemeral_id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Ephemeral identifier of this service (if one exists).\n\nThis id normally changes across restarts, but `service.id` does not.', + example: '8a4f500f', + }, + { + name: 'id', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'Unique identifier of the running service. If the service is comprised\nof many nodes, the `service.id` should be the same for all nodes.\n\nThis id should uniquely identify the service. This makes it possible to correlate\nlogs and metrics for one specific service, no matter which particular node\nemitted the event.\n\nNote that if you need to see the events from one specific host of the service,\nyou should filter on that `host.name` or `host.id` instead.', + example: 'd37e5ebfe0ae6c4972dbe9f0174a1637bb8247f6', + }, + { + name: 'name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the service data is collected from.\n\nThe name of the service is normally user given. This allows for distributed\nservices that run on multiple hosts to correlate the related instances based\non the name.\n\nIn the case of Elasticsearch the `service.name` could contain the cluster\nname. For Beats the `service.name` is by default a copy of the `service.type`\nfield if no name is specified.', + example: 'elasticsearch-metrics', + }, + { + name: 'node.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of a service node.\n\nThis allows for two nodes of the same service running on the same host to\nbe differentiated. Therefore, `service.node.name` should typically be unique\nacross nodes of a given service.\n\nIn the case of Elasticsearch, the `service.node.name` could contain the unique\nnode name within the Elasticsearch cluster. In cases where the service doe not\nhave the concept of a node name, the host name or container name can be used\nto distinguish running instances that make up this service. If those do not\nprovide uniqueness (e.g. multiple instances of the service running on the\nsame host) - the node name can be manually set.', + example: 'instance-0000000016', + }, + { + name: 'state', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Current state of the service.', + }, + { + name: 'type', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'The type of the service data is collected from.\n\nThe type can be used to group and correlate logs and metrics from one service\ntype.\n\nExample: If logs or metrics are collected from Elasticsearch, `service.type`\nwould be `elasticsearch`.', + example: 'elasticsearch', + }, + { + name: 'version', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: + 'Version of the service the data was collected from.\n\nThis allows to look at a data set only for a specific version of a service.', + example: '3.2.4', + }, + ], + }, + { + name: 'source', + title: 'Source', + group: 2, + description: + 'Source fields describe details about the source of a packet/event.\n\nSource fields are usually populated in conjunction with destination fields.', + type: 'group', + fields: [ + { + name: 'address', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Some event source addresses are defined ambiguously. The event\nwill sometimes list an IP, a domain or a unix socket. You should always store\nthe raw address in the `.address` field.\n\nThen it should be duplicated to `.ip` or `.domain`, depending on which one\nit is.', + }, + { + name: 'as.number', + level: 'extended', + type: 'long', + description: + 'Unique number allocated to the autonomous system. The autonomous\nsystem number (ASN) uniquely identifies each network on the Internet.', + example: 15169, + }, + { + name: 'as.organization.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Organization name.', + example: 'Google LLC', + }, + { + name: 'bytes', + level: 'core', + type: 'long', + format: 'bytes', + description: 'Bytes sent from the source to the destination.', + example: 184, + }, + { + name: 'domain', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Source domain.', + }, + { + name: 'geo.city_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'City name.', + example: 'Montreal', + }, + { + name: 'geo.continent_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the continent.', + example: 'North America', + }, + { + name: 'geo.country_iso_code', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Country ISO code.', + example: 'CA', + }, + { + name: 'geo.country_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Country name.', + example: 'Canada', + }, + { + name: 'geo.location', + level: 'core', + type: 'geo_point', + description: 'Longitude and latitude.', + example: '{ "lon": -73.614830, "lat": 45.505918 }', + }, + { + name: 'geo.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'User-defined description of a location, at the level of granularity\nthey care about.\n\nCould be the name of their data centers, the floor number, if this describes\na local physical entity, city names.\n\nNot typically used in automated geolocation.', + example: 'boston-dc', + }, + { + name: 'geo.region_iso_code', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Region ISO code.', + example: 'CA-QC', + }, + { + name: 'geo.region_name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Region name.', + example: 'Quebec', + }, + { + name: 'ip', + level: 'core', + type: 'ip', + description: + 'IP address of the source.\n\nCan be one or multiple IPv4 or IPv6 addresses.', + }, + { + name: 'mac', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'MAC address of the source.', + }, + { + name: 'nat.ip', + level: 'extended', + type: 'ip', + description: + 'Translated ip of source based NAT sessions (e.g. internal client\nto internet)\n\nTypically connections traversing load balancers, firewalls, or routers.', + }, + { + name: 'nat.port', + level: 'extended', + type: 'long', + format: 'string', + description: + 'Translated port of source based NAT sessions. (e.g. internal client\nto internet)\n\nTypically used with load balancers, firewalls, or routers.', + }, + { + name: 'packets', + level: 'core', + type: 'long', + description: 'Packets sent from the source to the destination.', + example: 12, + }, + { + name: 'port', + level: 'core', + type: 'long', + format: 'string', + description: 'Port of the source.', + }, + { + name: 'registered_domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The highest registered source domain, stripped of the subdomain.\n\nFor example, the registered domain for "foo.google.com" is "google.com".\n\nThis value can be determined precisely with a list like the public suffix\nlist (http://publicsuffix.org). Trying to approximate this by simply taking\nthe last two labels will not work well for TLDs such as "co.uk".', + example: 'google.com', + }, + { + name: 'top_level_domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The effective top level domain (eTLD), also known as the domain\nsuffix, is the last part of the domain name. For example, the top level domain\nfor google.com is "com".\n\nThis value can be determined precisely with a list like the public suffix\nlist (http://publicsuffix.org). Trying to approximate this by simply taking\nthe last label will not work well for effective TLDs such as "co.uk".', + example: 'co.uk', + }, + { + name: 'user.domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the directory the user is a member of.\n\nFor example, an LDAP or Active Directory domain name.', + }, + { + name: 'user.email', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'User email address.', + }, + { + name: 'user.full_name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: "User's full name, if available.", + example: 'Albert Einstein', + }, + { + name: 'user.group.domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the directory the group is a member of.\n\nFor example, an LDAP or Active Directory domain name.', + }, + { + name: 'user.group.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifier for the group on the system/platform.', + }, + { + name: 'user.group.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the group.', + }, + { + name: 'user.hash', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Unique user hash to correlate information for a user in anonymized\nform.\n\nUseful if `user.id` or `user.name` contain confidential information and cannot\nbe used.', + }, + { + name: 'user.id', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifiers of the user.', + }, + { + name: 'user.name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Short name or login of the user.', + example: 'albert', + }, + ], + }, + { + name: 'threat', + title: 'Threat', + group: 2, + description: + 'Fields to classify events and alerts according to a threat taxonomy\nsuch as the Mitre ATT&CK framework.\n\nThese fields are for users to classify alerts from all of their sources (e.g.\nIDS, NGFW, etc.) within a common taxonomy. The threat.tactic.* are meant to\ncapture the high level category of the threat (e.g. "impact"). The threat.technique.*\nfields are meant to capture which kind of approach is used by this detected\nthreat, to accomplish the goal (e.g. "endpoint denial of service").', + type: 'group', + fields: [ + { + name: 'framework', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the threat framework used to further categorize and classify\nthe tactic and technique of the reported threat. Framework classification\ncan be provided by detecting systems, evaluated at ingest time, or retrospectively\ntagged to events.', + example: 'MITRE ATT&CK', + }, + { + name: 'tactic.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The id of tactic used by this threat. You can use the Mitre ATT&CK\nMatrix Tactic categorization, for example. (ex. https://attack.mitre.org/tactics/TA0040/\n)', + example: 'TA0040', + }, + { + name: 'tactic.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the type of tactic used by this threat. You can use the\nMitre ATT&CK Matrix Tactic categorization, for example. (ex. https://attack.mitre.org/tactics/TA0040/\n)', + example: 'impact', + }, + { + name: 'tactic.reference', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The reference url of tactic used by this threat. You can use the\nMitre ATT&CK Matrix Tactic categorization, for example. (ex. https://attack.mitre.org/tactics/TA0040/\n)', + example: 'https://attack.mitre.org/tactics/TA0040/', + }, + { + name: 'technique.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The id of technique used by this tactic. You can use the Mitre\nATT&CK Matrix Tactic categorization, for example. (ex. https://attack.mitre.org/techniques/T1499/\n)', + example: 'T1499', + }, + { + name: 'technique.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: + 'The name of technique used by this tactic. You can use the Mitre\nATT&CK Matrix Tactic categorization, for example. (ex. https://attack.mitre.org/techniques/T1499/\n)', + example: 'endpoint denial of service', + }, + { + name: 'technique.reference', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The reference url of technique used by this tactic. You can use\nthe Mitre ATT&CK Matrix Tactic categorization, for example. (ex. https://attack.mitre.org/techniques/T1499/\n)', + example: 'https://attack.mitre.org/techniques/T1499/', + }, + ], + }, + { + name: 'tls', + title: 'TLS', + group: 2, + description: + 'Fields related to a TLS connection. These fields focus on the TLS\nprotocol itself and intentionally avoids in-depth analysis of the related x.509\ncertificate files.', + type: 'group', + fields: [ + { + name: 'cipher', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'String indicating the cipher used during the current connection.', + example: 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256', + default_field: false, + }, + { + name: 'client.certificate', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'PEM-encoded stand-alone certificate offered by the client. This\nis usually mutually-exclusive of `client.certificate_chain` since this value\nalso exists in that list.', + example: 'MII...', + default_field: false, + }, + { + name: 'client.certificate_chain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Array of PEM-encoded certificates that make up the certificate\nchain offered by the client. This is usually mutually-exclusive of `client.certificate`\nsince that value should be the first certificate in the chain.', + example: ['MII...', 'MII...'], + default_field: false, + }, + { + name: 'client.hash.md5', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Certificate fingerprint using the MD5 digest of DER-encoded version\nof certificate offered by the client. For consistency with other hash values,\nthis value should be formatted as an uppercase hash.', + example: '0F76C7F2C55BFD7D8E8B8F4BFBF0C9EC', + default_field: false, + }, + { + name: 'client.hash.sha1', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Certificate fingerprint using the SHA1 digest of DER-encoded version\nof certificate offered by the client. For consistency with other hash values,\nthis value should be formatted as an uppercase hash.', + example: '9E393D93138888D288266C2D915214D1D1CCEB2A', + default_field: false, + }, + { + name: 'client.hash.sha256', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Certificate fingerprint using the SHA256 digest of DER-encoded\nversion of certificate offered by the client. For consistency with other hash\nvalues, this value should be formatted as an uppercase hash.', + example: '0687F666A054EF17A08E2F2162EAB4CBC0D265E1D7875BE74BF3C712CA92DAF0', + default_field: false, + }, + { + name: 'client.issuer', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Distinguished name of subject of the issuer of the x.509 certificate\npresented by the client.', + example: 'CN=MyDomain Root CA, OU=Infrastructure Team, DC=mydomain, DC=com', + default_field: false, + }, + { + name: 'client.ja3', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'A hash that identifies clients based on how they perform an SSL/TLS\nhandshake.', + example: 'd4e5b18d6b55c71272893221c96ba240', + default_field: false, + }, + { + name: 'client.not_after', + level: 'extended', + type: 'date', + description: + 'Date/Time indicating when client certificate is no longer considered\nvalid.', + example: '2021-01-01T00:00:00.000Z', + default_field: false, + }, + { + name: 'client.not_before', + level: 'extended', + type: 'date', + description: 'Date/Time indicating when client certificate is first considered\nvalid.', + example: '1970-01-01T00:00:00.000Z', + default_field: false, + }, + { + name: 'client.server_name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Also called an SNI, this tells the server which hostname to which\nthe client is attempting to connect. When this value is available, it should\nget copied to `destination.domain`.', + example: 'www.elastic.co', + default_field: false, + }, + { + name: 'client.subject', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Distinguished name of subject of the x.509 certificate presented\nby the client.', + example: 'CN=myclient, OU=Documentation Team, DC=mydomain, DC=com', + default_field: false, + }, + { + name: 'client.supported_ciphers', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Array of ciphers offered by the client during the client hello.', + example: [ + 'TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384', + 'TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384', + '...', + ], + default_field: false, + }, + { + name: 'curve', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'String indicating the curve used for the given cipher, when applicable.', + example: 'secp256r1', + default_field: false, + }, + { + name: 'established', + level: 'extended', + type: 'boolean', + description: + 'Boolean flag indicating if the TLS negotiation was successful and\ntransitioned to an encrypted tunnel.', + default_field: false, + }, + { + name: 'next_protocol', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'String indicating the protocol being tunneled. Per the values in\nthe IANA registry (https://www.iana.org/assignments/tls-extensiontype-values/tls-extensiontype-values.xhtml#alpn-protocol-ids),\nthis string should be lower case.', + example: 'http/1.1', + default_field: false, + }, + { + name: 'resumed', + level: 'extended', + type: 'boolean', + description: + 'Boolean flag indicating if this TLS connection was resumed from\nan existing TLS negotiation.', + default_field: false, + }, + { + name: 'server.certificate', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'PEM-encoded stand-alone certificate offered by the server. This\nis usually mutually-exclusive of `server.certificate_chain` since this value\nalso exists in that list.', + example: 'MII...', + default_field: false, + }, + { + name: 'server.certificate_chain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Array of PEM-encoded certificates that make up the certificate\nchain offered by the server. This is usually mutually-exclusive of `server.certificate`\nsince that value should be the first certificate in the chain.', + example: ['MII...', 'MII...'], + default_field: false, + }, + { + name: 'server.hash.md5', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Certificate fingerprint using the MD5 digest of DER-encoded version\nof certificate offered by the server. For consistency with other hash values,\nthis value should be formatted as an uppercase hash.', + example: '0F76C7F2C55BFD7D8E8B8F4BFBF0C9EC', + default_field: false, + }, + { + name: 'server.hash.sha1', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Certificate fingerprint using the SHA1 digest of DER-encoded version\nof certificate offered by the server. For consistency with other hash values,\nthis value should be formatted as an uppercase hash.', + example: '9E393D93138888D288266C2D915214D1D1CCEB2A', + default_field: false, + }, + { + name: 'server.hash.sha256', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Certificate fingerprint using the SHA256 digest of DER-encoded\nversion of certificate offered by the server. For consistency with other hash\nvalues, this value should be formatted as an uppercase hash.', + example: '0687F666A054EF17A08E2F2162EAB4CBC0D265E1D7875BE74BF3C712CA92DAF0', + default_field: false, + }, + { + name: 'server.issuer', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Subject of the issuer of the x.509 certificate presented by the\nserver.', + example: 'CN=MyDomain Root CA, OU=Infrastructure Team, DC=mydomain, DC=com', + default_field: false, + }, + { + name: 'server.ja3s', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'A hash that identifies servers based on how they perform an SSL/TLS\nhandshake.', + example: '394441ab65754e2207b1e1b457b3641d', + default_field: false, + }, + { + name: 'server.not_after', + level: 'extended', + type: 'date', + description: + 'Timestamp indicating when server certificate is no longer considered\nvalid.', + example: '2021-01-01T00:00:00.000Z', + default_field: false, + }, + { + name: 'server.not_before', + level: 'extended', + type: 'date', + description: 'Timestamp indicating when server certificate is first considered\nvalid.', + example: '1970-01-01T00:00:00.000Z', + default_field: false, + }, + { + name: 'server.subject', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Subject of the x.509 certificate presented by the server.', + example: 'CN=www.mydomain.com, OU=Infrastructure Team, DC=mydomain, DC=com', + default_field: false, + }, + { + name: 'version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Numeric part of the version parsed from the original string.', + example: '1.2', + default_field: false, + }, + { + name: 'version_protocol', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Normalized lowercase protocol name parsed from original string.', + example: 'tls', + default_field: false, + }, + ], + }, + { + name: 'tracing', + title: 'Tracing', + group: 2, + description: + 'Distributed tracing makes it possible to analyze performance throughout\na microservice architecture all in one view. This is accomplished by tracing\nall of the requests - from the initial web request in the front-end service\n- to queries made through multiple back-end services.', + type: 'group', + fields: [ + { + name: 'trace.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Unique identifier of the trace.\n\nA trace groups multiple events like transactions that belong together. For\nexample, a user request handled by multiple inter-connected services.', + example: '4bf92f3577b34da6a3ce929d0e0e4736', + }, + { + name: 'transaction.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Unique identifier of the transaction.\n\nA transaction is the highest level of work measured within a service, such\nas a request to a server.', + example: '00f067aa0ba902b7', + }, + ], + }, + { + name: 'url', + title: 'URL', + group: 2, + description: + 'URL fields provide support for complete or partial URLs, and supports\nthe breaking down into scheme, domain, path, and so on.', + type: 'group', + fields: [ + { + name: 'domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Domain of the url, such as "www.elastic.co".\n\nIn some cases a URL may refer to an IP and/or port directly, without a domain\nname. In this case, the IP address would go to the `domain` field.', + example: 'www.elastic.co', + }, + { + name: 'extension', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The field contains the file extension from the original request\nurl.\n\nThe file extension is only set if it exists, as not every url has a file extension.\n\nThe leading period must not be included. For example, the value must be "png",\nnot ".png".', + example: 'png', + }, + { + name: 'fragment', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Portion of the url after the `#`, such as "top".\n\nThe `#` is not part of the fragment.', + }, + { + name: 'full', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: + 'If full URLs are important to your use case, they should be stored\nin `url.full`, whether this field is reconstructed or present in the event\nsource.', + example: 'https://www.elastic.co:443/search?q=elasticsearch#top', + }, + { + name: 'original', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: + 'Unmodified original url as seen in the event source.\n\nNote that in network monitoring, the observed URL may be a full URL, whereas\nin access logs, the URL is often just represented as a path.\n\nThis field is meant to represent the URL as it was observed, complete or not.', + example: + 'https://www.elastic.co:443/search?q=elasticsearch#top or /search?q=elasticsearch', + }, + { + name: 'password', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Password of the request.', + }, + { + name: 'path', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Path of the request, such as "/search".', + }, + { + name: 'port', + level: 'extended', + type: 'long', + format: 'string', + description: 'Port of the request, such as 443.', + example: 443, + }, + { + name: 'query', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The query field describes the query string of the request, such\nas "q=elasticsearch".\n\nThe `?` is excluded from the query string. If a URL contains no `?`, there\nis no query field. If there is a `?` but no query, the query field exists\nwith an empty string. The `exists` query can be used to differentiate between\nthe two cases.', + }, + { + name: 'registered_domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The highest registered url domain, stripped of the subdomain.\n\nFor example, the registered domain for "foo.google.com" is "google.com".\n\nThis value can be determined precisely with a list like the public suffix\nlist (http://publicsuffix.org). Trying to approximate this by simply taking\nthe last two labels will not work well for TLDs such as "co.uk".', + example: 'google.com', + }, + { + name: 'scheme', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Scheme of the request, such as "https".\n\nNote: The `:` is not part of the scheme.', + example: 'https', + }, + { + name: 'top_level_domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The effective top level domain (eTLD), also known as the domain\nsuffix, is the last part of the domain name. For example, the top level domain\nfor google.com is "com".\n\nThis value can be determined precisely with a list like the public suffix\nlist (http://publicsuffix.org). Trying to approximate this by simply taking\nthe last label will not work well for effective TLDs such as "co.uk".', + example: 'co.uk', + }, + { + name: 'username', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Username of the request.', + }, + ], + }, + { + name: 'user', + title: 'User', + group: 2, + description: + 'The user fields describe information about the user that is relevant\nto the event.\n\nFields can have one entry or multiple entries. If a user has more than one id,\nprovide an array that includes all of them.', + type: 'group', + fields: [ + { + name: 'domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the directory the user is a member of.\n\nFor example, an LDAP or Active Directory domain name.', + }, + { + name: 'email', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'User email address.', + }, + { + name: 'full_name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: "User's full name, if available.", + example: 'Albert Einstein', + }, + { + name: 'group.domain', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Name of the directory the group is a member of.\n\nFor example, an LDAP or Active Directory domain name.', + }, + { + name: 'group.id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifier for the group on the system/platform.', + }, + { + name: 'group.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the group.', + }, + { + name: 'hash', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'Unique user hash to correlate information for a user in anonymized\nform.\n\nUseful if `user.id` or `user.name` contain confidential information and cannot\nbe used.', + }, + { + name: 'id', + level: 'core', + type: 'keyword', + ignore_above: 1024, + description: 'Unique identifiers of the user.', + }, + { + name: 'name', + level: 'core', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Short name or login of the user.', + example: 'albert', + }, + ], + }, + { + name: 'user_agent', + title: 'User agent', + group: 2, + description: + 'The user_agent fields normally come from a browser request.\n\nThey often show up in web service logs coming from the parsed user agent string.', + type: 'group', + fields: [ + { + name: 'device.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the device.', + example: 'iPhone', + }, + { + name: 'name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Name of the user agent.', + example: 'Safari', + }, + { + name: 'original', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + }, + ], + description: 'Unparsed user_agent string.', + example: + 'Mozilla/5.0 (iPhone; CPU iPhone OS 12_1 like Mac OS X) AppleWebKit/605.1.15\n(KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1', + }, + { + name: 'os.family', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'OS family (such as redhat, debian, freebsd, windows).', + example: 'debian', + }, + { + name: 'os.full', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Operating system name, including the version or code name.', + example: 'Mac OS Mojave', + }, + { + name: 'os.kernel', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Operating system kernel version as a raw string.', + example: '4.4.0-112-generic', + }, + { + name: 'os.name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + default_field: false, + }, + ], + description: 'Operating system name, without the version.', + example: 'Mac OS X', + }, + { + name: 'os.platform', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Operating system platform (such centos, ubuntu, windows).', + example: 'darwin', + }, + { + name: 'os.version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Operating system version as a raw string.', + example: '10.14.1', + }, + { + name: 'version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Version of the user agent.', + example: 12, + }, + ], + }, + { + name: 'vlan', + title: 'VLAN', + group: 2, + description: + 'The VLAN fields are used to identify 802.1q tag(s) of a packet,\nas well as ingress and egress VLAN associations of an observer in relation to\na specific packet or connection.\n\nNetwork.vlan fields are used to record a single VLAN tag, or the outer tag in\nthe case of q-in-q encapsulations, for a packet or connection as observed, typically\nprovided by a network sensor (e.g. Zeek, Wireshark) passively reporting on traffic.\n\nNetwork.inner VLAN fields are used to report inner q-in-q 802.1q tags (multiple\n802.1q encapsulations) as observed, typically provided by a network sensor (e.g.\nZeek, Wireshark) passively reporting on traffic. Network.inner VLAN fields should\nonly be used in addition to network.vlan fields to indicate q-in-q tagging.\n\nObserver.ingress and observer.egress VLAN values are used to record observer\nspecific information when observer events contain discrete ingress and egress\nVLAN information, typically provided by firewalls, routers, or load balancers.', + type: 'group', + fields: [ + { + name: 'id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'VLAN ID as reported by the observer.', + example: 10, + default_field: false, + }, + { + name: 'name', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'Optional VLAN name as reported by the observer.', + example: 'outside', + default_field: false, + }, + ], + }, + { + name: 'vulnerability', + title: 'Vulnerability', + group: 2, + description: + 'The vulnerability fields describe information about a vulnerability\nthat is relevant to an event.', + type: 'group', + fields: [ + { + name: 'category', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The type of system or architecture that the vulnerability affects.\nThese may be platform-specific (for example, Debian or SUSE) or general (for\nexample, Database or Firewall). For example (https://qualysguard.qualys.com/qwebhelp/fo_portal/knowledgebase/vulnerability_categories.htm[Qualys\nvulnerability categories])\n\nThis field must be an array.', + example: '["Firewall"]', + default_field: false, + }, + { + name: 'classification', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The classification of the vulnerability scoring system. For example\n(https://www.first.org/cvss/)', + example: 'CVSS', + default_field: false, + }, + { + name: 'description', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + multi_fields: [ + { + name: 'text', + type: 'text', + norms: false, + }, + ], + description: + 'The description of the vulnerability that provides additional context\nof the vulnerability. For example (https://cve.mitre.org/about/faqs.html#cve_entry_descriptions_created[Common\nVulnerabilities and Exposure CVE description])', + example: 'In macOS before 2.12.6, there is a vulnerability in the RPC...', + default_field: false, + }, + { + name: 'enumeration', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The type of identifier used for this vulnerability. For example\n(https://cve.mitre.org/about/)', + example: 'CVE', + default_field: false, + }, + { + name: 'id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The identification (ID) is the number portion of a vulnerability\nentry. It includes a unique identification number for the vulnerability. For\nexample (https://cve.mitre.org/about/faqs.html#what_is_cve_id)[Common Vulnerabilities\nand Exposure CVE ID]', + example: 'CVE-2019-00001', + default_field: false, + }, + { + name: 'reference', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'A resource that provides additional information, context, and mitigations\nfor the identified vulnerability.', + example: 'https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-6111', + default_field: false, + }, + { + name: 'report_id', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'The report or scan identification number.', + example: 20191018.0001, + default_field: false, + }, + { + name: 'scanner.vendor', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: 'The name of the vulnerability scanner vendor.', + example: 'Tenable', + default_field: false, + }, + { + name: 'score.base', + level: 'extended', + type: 'float', + description: + 'Scores can range from 0.0 to 10.0, with 10.0 being the most severe.\n\nBase scores cover an assessment for exploitability metrics (attack vector,\ncomplexity, privileges, and user interaction), impact metrics (confidentiality,\nintegrity, and availability), and scope. For example (https://www.first.org/cvss/specification-document)', + example: 5.5, + default_field: false, + }, + { + name: 'score.environmental', + level: 'extended', + type: 'float', + description: + 'Scores can range from 0.0 to 10.0, with 10.0 being the most severe.\n\nEnvironmental scores cover an assessment for any modified Base metrics, confidentiality,\nintegrity, and availability requirements. For example (https://www.first.org/cvss/specification-document)', + example: 5.5, + default_field: false, + }, + { + name: 'score.temporal', + level: 'extended', + type: 'float', + description: + 'Scores can range from 0.0 to 10.0, with 10.0 being the most severe.\n\nTemporal scores cover an assessment for code maturity, remediation level,\nand confidence. For example (https://www.first.org/cvss/specification-document)', + default_field: false, + }, + { + name: 'score.version', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The National Vulnerability Database (NVD) provides qualitative\nseverity rankings of "Low", "Medium", and "High" for CVSS v2.0 base score\nranges in addition to the severity ratings for CVSS v3.0 as they are defined\nin the CVSS v3.0 specification.\n\nCVSS is owned and managed by FIRST.Org, Inc. (FIRST), a US-based non-profit\norganization, whose mission is to help computer security incident response\nteams across the world. For example (https://nvd.nist.gov/vuln-metrics/cvss)', + example: 2, + default_field: false, + }, + { + name: 'severity', + level: 'extended', + type: 'keyword', + ignore_above: 1024, + description: + 'The severity of the vulnerability can help with metrics and internal\nprioritization regarding remediation. For example (https://nvd.nist.gov/vuln-metrics/cvss)', + example: 'Critical', + default_field: false, + }, + ], + }, + ], + }, + { + key: 'beat', + anchor: 'beat-common', + title: 'Beat', + description: 'Contains common beat fields available in all event types.\n', + fields: [ + { + name: 'agent.hostname', + type: 'keyword', + description: 'Hostname of the agent.', + }, + { + name: 'beat.timezone', + type: 'alias', + path: 'event.timezone', + migration: true, + }, + { + name: 'fields', + type: 'object', + object_type: 'keyword', + description: 'Contains user configurable fields.\n', + }, + { + name: 'beat.name', + type: 'alias', + path: 'host.name', + migration: true, + }, + { + name: 'beat.hostname', + type: 'alias', + path: 'agent.hostname', + migration: true, + }, + { + name: 'timeseries.instance', + type: 'keyword', + description: 'Time series instance id', + }, + ], + }, + { + key: 'cloud', + title: 'Cloud provider metadata', + description: 'Metadata from cloud providers added by the add_cloud_metadata processor.\n', + fields: [ + { + name: 'cloud.project.id', + example: 'project-x', + description: 'Name of the project in Google Cloud.\n', + }, + { + name: 'cloud.image.id', + example: 'ami-abcd1234', + description: 'Image ID for the cloud instance.\n', + }, + { + name: 'meta.cloud.provider', + type: 'alias', + path: 'cloud.provider', + migration: true, + }, + { + name: 'meta.cloud.instance_id', + type: 'alias', + path: 'cloud.instance.id', + migration: true, + }, + { + name: 'meta.cloud.instance_name', + type: 'alias', + path: 'cloud.instance.name', + migration: true, + }, + { + name: 'meta.cloud.machine_type', + type: 'alias', + path: 'cloud.machine.type', + migration: true, + }, + { + name: 'meta.cloud.availability_zone', + type: 'alias', + path: 'cloud.availability_zone', + migration: true, + }, + { + name: 'meta.cloud.project_id', + type: 'alias', + path: 'cloud.project.id', + migration: true, + }, + { + name: 'meta.cloud.region', + type: 'alias', + path: 'cloud.region', + migration: true, + }, + ], + }, + { + key: 'docker', + title: 'Docker', + description: 'Docker stats collected from Docker.\n', + short_config: false, + anchor: 'docker-processor', + fields: [ + { + name: 'docker', + type: 'group', + fields: [ + { + name: 'container.id', + type: 'alias', + path: 'container.id', + migration: true, + }, + { + name: 'container.image', + type: 'alias', + path: 'container.image.name', + migration: true, + }, + { + name: 'container.name', + type: 'alias', + path: 'container.name', + migration: true, + }, + { + name: 'container.labels', + type: 'object', + object_type: 'keyword', + description: 'Image labels.\n', + }, + ], + }, + ], + }, + { + key: 'host', + title: 'Host', + description: 'Info collected for the host machine.\n', + anchor: 'host-processor', + fields: [ + { + name: 'host', + type: 'group', + fields: [ + { + name: 'containerized', + type: 'boolean', + description: 'If the host is a container.\n', + }, + { + name: 'os.build', + type: 'keyword', + example: '18D109', + description: 'OS build information.\n', + }, + { + name: 'os.codename', + type: 'keyword', + example: 'stretch', + description: 'OS codename, if any.\n', + }, + ], + }, + ], + }, + { + key: 'kubernetes', + title: 'Kubernetes', + description: 'Kubernetes metadata added by the kubernetes processor\n', + short_config: false, + anchor: 'kubernetes-processor', + fields: [ + { + name: 'kubernetes', + type: 'group', + fields: [ + { + name: 'pod.name', + type: 'keyword', + description: 'Kubernetes pod name\n', + }, + { + name: 'pod.uid', + type: 'keyword', + description: 'Kubernetes Pod UID\n', + }, + { + name: 'namespace', + type: 'keyword', + description: 'Kubernetes namespace\n', + }, + { + name: 'node.name', + type: 'keyword', + description: 'Kubernetes node name\n', + }, + { + name: 'labels.*', + type: 'object', + object_type: 'keyword', + object_type_mapping_type: '*', + description: 'Kubernetes labels map\n', + }, + { + name: 'annotations.*', + type: 'object', + object_type: 'keyword', + object_type_mapping_type: '*', + description: 'Kubernetes annotations map\n', + }, + { + name: 'replicaset.name', + type: 'keyword', + description: 'Kubernetes replicaset name\n', + }, + { + name: 'deployment.name', + type: 'keyword', + description: 'Kubernetes deployment name\n', + }, + { + name: 'statefulset.name', + type: 'keyword', + description: 'Kubernetes statefulset name\n', + }, + { + name: 'container.name', + type: 'keyword', + description: 'Kubernetes container name\n', + }, + { + name: 'container.image', + type: 'keyword', + description: 'Kubernetes container image\n', + }, + ], + }, + ], + }, + { + key: 'process', + title: 'Process', + description: 'Process metadata fields\n', + fields: [ + { + name: 'process', + type: 'group', + fields: [ + { + name: 'exe', + type: 'alias', + path: 'process.executable', + migration: true, + }, + ], + }, + ], + }, + { + key: 'jolokia-autodiscover', + title: 'Jolokia Discovery autodiscover provider', + description: 'Metadata from Jolokia Discovery added by the jolokia provider.\n', + fields: [ + { + name: 'jolokia.agent.version', + type: 'keyword', + description: 'Version number of jolokia agent.\n', + }, + { + name: 'jolokia.agent.id', + type: 'keyword', + description: + 'Each agent has a unique id which can be either provided during startup of the agent in form of a configuration parameter or being autodetected. If autodected, the id has several parts: The IP, the process id, hashcode of the agent and its type.\n', + }, + { + name: 'jolokia.server.product', + type: 'keyword', + description: 'The container product if detected.\n', + }, + { + name: 'jolokia.server.version', + type: 'keyword', + description: "The container's version (if detected).\n", + }, + { + name: 'jolokia.server.vendor', + type: 'keyword', + description: 'The vendor of the container the agent is running in.\n', + }, + { + name: 'jolokia.url', + type: 'keyword', + description: 'The URL how this agent can be contacted.\n', + }, + { + name: 'jolokia.secured', + type: 'boolean', + description: 'Whether the agent was configured for authentication or not.\n', + }, + ], + }, + { + key: 'common', + title: 'Common', + description: + 'These fields contain data about the environment in which the transaction or flow was captured.\n', + fields: [ + { + name: 'type', + description: + 'The type of the transaction (for example, HTTP, MySQL, Redis, or RUM) or "flow" in case of flows.\n', + required: true, + }, + { + name: 'server.process.name', + description: 'The name of the process that served the transaction.\n', + }, + { + name: 'server.process.args', + description: 'The command-line of the process that served the transaction.\n', + }, + { + name: 'server.process.executable', + description: 'Absolute path to the server process executable.\n', + }, + { + name: 'server.process.working_directory', + description: 'The working directory of the server process.\n', + }, + { + name: 'server.process.start', + description: 'The time the server process started.\n', + }, + { + name: 'client.process.name', + description: 'The name of the process that initiated the transaction.\n', + }, + { + name: 'client.process.args', + description: 'The command-line of the process that initiated the transaction.\n', + }, + { + name: 'client.process.executable', + description: 'Absolute path to the client process executable.\n', + }, + { + name: 'client.process.working_directory', + description: 'The working directory of the client process.\n', + }, + { + name: 'client.process.start', + description: 'The time the client process started.\n', + }, + { + name: 'real_ip', + type: 'alias', + path: 'network.forwarded_ip', + migration: true, + description: + 'If the server initiating the transaction is a proxy, this field contains the original client IP address. For HTTP, for example, the IP address extracted from a configurable HTTP header, by default `X-Forwarded-For`.\nUnless this field is disabled, it always has a value, and it matches the `client_ip` for non proxy clients.\n', + }, + { + name: 'transport', + type: 'alias', + path: 'network.transport', + migration: true, + description: + 'The transport protocol used for the transaction. If not specified, then tcp is assumed.\n', + }, + ], + }, + { + key: 'flows_event', + title: 'Flow Event', + description: 'These fields contain data about the flow itself.\n', + fields: [ + { + name: 'flow.final', + type: 'boolean', + description: + 'Indicates if event is last event in flow. If final is false, the event reports an intermediate flow state only.\n', + }, + { + name: 'flow.id', + description: 'Internal flow ID based on connection meta data and address.\n', + }, + { + name: 'flow.vlan', + type: 'long', + description: + "VLAN identifier from the 802.1q frame. In case of a multi-tagged frame this field will be an array with the outer tag's VLAN identifier listed first.\n", + }, + { + name: 'flow_id', + type: 'alias', + path: 'flow.id', + migration: true, + }, + { + name: 'final', + type: 'alias', + path: 'flow.final', + migration: true, + }, + { + name: 'vlan', + type: 'alias', + path: 'flow.vlan', + migration: true, + }, + { + name: 'source.stats.net_bytes_total', + type: 'alias', + path: 'source.bytes', + migration: true, + }, + { + name: 'source.stats.net_packets_total', + type: 'alias', + path: 'source.packets', + migration: true, + }, + { + name: 'dest.stats.net_bytes_total', + type: 'alias', + path: 'destination.bytes', + migration: true, + }, + { + name: 'dest.stats.net_packets_total', + type: 'alias', + path: 'destination.packets', + migration: true, + }, + ], + }, + { + key: 'trans_event', + title: 'Transaction Event', + description: 'These fields contain data about the transaction itself.\n', + fields: [ + { + name: 'status', + description: + 'The high level status of the transaction. The way to compute this value depends on the protocol, but the result has a meaning independent of the protocol.\n', + required: true, + possible_values: ['OK', 'Error', 'Server Error', 'Client Error'], + }, + { + name: 'method', + description: + 'The command/verb/method of the transaction. For HTTP, this is the method name (GET, POST, PUT, and so on), for SQL this is the verb (SELECT, UPDATE, DELETE, and so on).\n', + }, + { + name: 'resource', + description: + 'The logical resource that this transaction refers to. For HTTP, this is the URL path up to the last slash (/). For example, if the URL is `/users/1`, the resource is `/users`. For databases, the resource is typically the table name. The field is not filled for all transaction types.\n', + }, + { + name: 'path', + required: true, + description: + 'The path the transaction refers to. For HTTP, this is the URL. For SQL databases, this is the table name. For key-value stores, this is the key.\n', + }, + { + name: 'query', + type: 'keyword', + description: + 'The query in a human readable format. For HTTP, it will typically be something like `GET /users/_search?name=test`. For MySQL, it is something like `SELECT id from users where name=test`.\n', + }, + { + name: 'params', + type: 'text', + description: + 'The request parameters. For HTTP, these are the POST or GET parameters. For Thrift-RPC, these are the parameters from the request.\n', + }, + { + name: 'notes', + type: 'alias', + path: 'error.message', + description: + 'Messages from Packetbeat itself. This field usually contains error messages for interpreting the raw data. This information can be helpful for troubleshooting.\n', + }, + ], + }, + { + key: 'raw', + title: 'Raw', + description: 'These fields contain the raw transaction data.', + fields: [ + { + name: 'request', + type: 'text', + description: + 'For text protocols, this is the request as seen on the wire (application layer only). For binary protocols this is our representation of the request.\n', + }, + { + name: 'response', + type: 'text', + description: + 'For text protocols, this is the response as seen on the wire (application layer only). For binary protocols this is our representation of the request.\n', + }, + ], + }, + { + key: 'trans_measurements', + title: 'Measurements (Transactions)', + description: 'These fields contain measurements related to the transaction.\n', + fields: [ + { + name: 'bytes_in', + type: 'alias', + path: 'source.bytes', + description: + 'The number of bytes of the request. Note that this size is the application layer message length, without the length of the IP or TCP headers.\n', + }, + { + name: 'bytes_out', + type: 'alias', + path: 'destination.bytes', + description: + 'The number of bytes of the response. Note that this size is the application layer message length, without the length of the IP or TCP headers.\n', + }, + ], + }, + { + key: 'amqp', + title: 'AMQP', + description: 'AMQP specific event fields.', + fields: [ + { + name: 'amqp', + type: 'group', + fields: [ + { + name: 'reply-code', + type: 'long', + description: 'AMQP reply code to an error, similar to http reply-code\n', + example: 404, + }, + { + name: 'reply-text', + type: 'keyword', + description: 'Text explaining the error.\n', + }, + { + name: 'class-id', + type: 'long', + description: 'Failing method class.\n', + }, + { + name: 'method-id', + type: 'long', + description: 'Failing method ID.\n', + }, + { + name: 'exchange', + type: 'keyword', + description: 'Name of the exchange.\n', + }, + { + name: 'exchange-type', + type: 'keyword', + description: 'Exchange type.\n', + example: 'fanout', + }, + { + name: 'passive', + type: 'boolean', + description: 'If set, do not create exchange/queue.\n', + }, + { + name: 'durable', + type: 'boolean', + description: 'If set, request a durable exchange/queue.\n', + }, + { + name: 'exclusive', + type: 'boolean', + description: 'If set, request an exclusive queue.\n', + }, + { + name: 'auto-delete', + type: 'boolean', + description: 'If set, auto-delete queue when unused.\n', + }, + { + name: 'no-wait', + type: 'boolean', + description: 'If set, the server will not respond to the method.\n', + }, + { + name: 'consumer-tag', + description: 'Identifier for the consumer, valid within the current channel.\n', + }, + { + name: 'delivery-tag', + type: 'long', + description: 'The server-assigned and channel-specific delivery tag.\n', + }, + { + name: 'message-count', + type: 'long', + description: + 'The number of messages in the queue, which will be zero for newly-declared queues.\n', + }, + { + name: 'consumer-count', + type: 'long', + description: 'The number of consumers of a queue.\n', + }, + { + name: 'routing-key', + type: 'keyword', + description: 'Message routing key.\n', + }, + { + name: 'no-ack', + type: 'boolean', + description: 'If set, the server does not expect acknowledgements for messages.\n', + }, + { + name: 'no-local', + type: 'boolean', + description: + 'If set, the server will not send messages to the connection that published them.\n', + }, + { + name: 'if-unused', + type: 'boolean', + description: 'Delete only if unused.\n', + }, + { + name: 'if-empty', + type: 'boolean', + description: 'Delete only if empty.\n', + }, + { + name: 'queue', + type: 'keyword', + description: 'The queue name identifies the queue within the vhost.\n', + }, + { + name: 'redelivered', + type: 'boolean', + description: + 'Indicates that the message has been previously delivered to this or another client.\n', + }, + { + name: 'multiple', + type: 'boolean', + description: 'Acknowledge multiple messages.\n', + }, + { + name: 'arguments', + type: 'object', + description: + 'Optional additional arguments passed to some methods. Can be of various types.\n', + }, + { + name: 'mandatory', + type: 'boolean', + description: 'Indicates mandatory routing.\n', + }, + { + name: 'immediate', + type: 'boolean', + description: 'Request immediate delivery.\n', + }, + { + name: 'content-type', + type: 'keyword', + description: 'MIME content type.\n', + example: 'text/plain', + }, + { + name: 'content-encoding', + type: 'keyword', + description: 'MIME content encoding.\n', + }, + { + name: 'headers', + type: 'object', + object_type: 'keyword', + description: 'Message header field table.\n', + }, + { + name: 'delivery-mode', + type: 'keyword', + description: 'Non-persistent (1) or persistent (2).\n', + }, + { + name: 'priority', + type: 'long', + description: 'Message priority, 0 to 9.\n', + }, + { + name: 'correlation-id', + type: 'keyword', + description: 'Application correlation identifier.\n', + }, + { + name: 'reply-to', + type: 'keyword', + description: 'Address to reply to.\n', + }, + { + name: 'expiration', + type: 'keyword', + description: 'Message expiration specification.\n', + }, + { + name: 'message-id', + type: 'keyword', + description: 'Application message identifier.\n', + }, + { + name: 'timestamp', + type: 'keyword', + description: 'Message timestamp.\n', + }, + { + name: 'type', + type: 'keyword', + description: 'Message type name.\n', + }, + { + name: 'user-id', + type: 'keyword', + description: 'Creating user id.\n', + }, + { + name: 'app-id', + type: 'keyword', + description: 'Creating application id.\n', + }, + ], + }, + ], + }, + { + key: 'cassandra', + title: 'Cassandra', + description: 'Cassandra v4/3 specific event fields.', + fields: [ + { + name: 'no_request', + type: 'alias', + path: 'cassandra.no_request', + migration: true, + }, + { + name: 'cassandra', + type: 'group', + description: 'Information about the Cassandra request and response.', + fields: [ + { + name: 'no_request', + type: 'boolean', + description: 'Indicates that there is no request because this is a PUSH message.\n', + }, + { + name: 'request', + type: 'group', + description: 'Cassandra request.', + fields: [ + { + name: 'headers', + type: 'group', + description: 'Cassandra request headers.', + fields: [ + { + name: 'version', + type: 'long', + description: 'The version of the protocol.', + }, + { + name: 'flags', + type: 'keyword', + description: 'Flags applying to this frame.', + }, + { + name: 'stream', + type: 'keyword', + description: + 'A frame has a stream id. If a client sends a request message with the stream id X, it is guaranteed that the stream id of the response to that message will be X.', + }, + { + name: 'op', + type: 'keyword', + description: 'An operation type that distinguishes the actual message.', + }, + { + name: 'length', + type: 'long', + description: + 'A integer representing the length of the body of the frame (a frame is limited to 256MB in length).', + }, + ], + }, + { + name: 'query', + type: 'keyword', + description: 'The CQL query which client send to cassandra.', + }, + ], + }, + { + name: 'response', type: 'group', description: 'Cassandra response.', fields: [ @@ -3243,19 +6791,19 @@ export const packetbeatSchema: Schema = [ name: 'transaction_id', type: 'keyword', description: - 'Transaction ID, a random number chosen by the client, used by the client and server to associate messages and responses between a client and a server.', + 'Transaction ID, a random number chosen by the\nclient, used by the client and server to associate\nmessages and responses between a client and a\nserver.\n', }, { name: 'seconds', type: 'long', description: - 'Number of seconds elapsed since client began address acquisition or renewal process.', + 'Number of seconds elapsed since client began address acquisition or\nrenewal process.\n', }, { name: 'flags', type: 'keyword', description: - 'Flags are set by the client to indicate how the DHCP server should its reply -- either unicast or broadcast.', + 'Flags are set by the client to indicate how the DHCP server should\nits reply -- either unicast or broadcast.\n', }, { name: 'client_ip', @@ -3266,19 +6814,19 @@ export const packetbeatSchema: Schema = [ name: 'assigned_ip', type: 'ip', description: - 'The IP address that the DHCP server is assigning to the client. This field is also known as "your" IP address.', + 'The IP address that the DHCP server is assigning to the client.\nThis field is also known as "your" IP address.\n', }, { name: 'server_ip', type: 'ip', description: - 'The IP address of the DHCP server that the client should use for the next step in the bootstrap process.', + 'The IP address of the DHCP server that the client should use for the\nnext step in the bootstrap process.\n', }, { name: 'relay_ip', type: 'ip', description: - 'The relay IP address used by the client to contact the server (i.e. a DHCP relay server).', + 'The relay IP address used by the client to contact the server\n(i.e. a DHCP relay server).\n', }, { name: 'client_mac', @@ -3289,13 +6837,13 @@ export const packetbeatSchema: Schema = [ name: 'server_name', type: 'keyword', description: - 'The name of the server sending the message. Optional. Used in DHCPOFFER or DHCPACK messages.', + 'The name of the server sending the message. Optional. Used in\nDHCPOFFER or DHCPACK messages.\n', }, { name: 'op_code', type: 'keyword', example: 'bootreply', - description: 'The message op code (bootrequest or bootreply).', + description: 'The message op code (bootrequest or bootreply).\n', }, { name: 'hops', @@ -3306,7 +6854,7 @@ export const packetbeatSchema: Schema = [ name: 'hardware_type', type: 'keyword', description: - 'The type of hardware used for the local network (Ethernet, LocalTalk, etc).', + 'The type of hardware used for the local network (Ethernet,\nLocalTalk, etc).\n', }, { name: 'option', @@ -3317,124 +6865,126 @@ export const packetbeatSchema: Schema = [ type: 'keyword', example: 'ack', description: - 'The specific type of DHCP message being sent (e.g. discover, offer, request, decline, ack, nak, release, inform).', + 'The specific type of DHCP message being sent (e.g. discover,\noffer, request, decline, ack, nak, release, inform).\n', }, { name: 'parameter_request_list', type: 'keyword', description: - 'This option is used by a DHCP client to request values for specified configuration parameters.', + 'This option is used by a DHCP client to request values for\nspecified configuration parameters.\n', }, { name: 'requested_ip_address', type: 'ip', description: - 'This option is used in a client request (DHCPDISCOVER) to allow the client to request that a particular IP address be assigned.', + 'This option is used in a client request (DHCPDISCOVER) to allow\nthe client to request that a particular IP address be assigned.\n', }, { name: 'server_identifier', type: 'ip', - description: 'IP address of the individual DHCP server which handled this message.', + description: + 'IP address of the individual DHCP server which handled this\nmessage.\n', }, { name: 'broadcast_address', type: 'ip', description: - "This option specifies the broadcast address in use on the client's subnet. ", + "This option specifies the broadcast address in use on the\nclient's subnet.\n", }, { name: 'max_dhcp_message_size', type: 'long', description: - 'This option specifies the maximum length DHCP message that the client is willing to accept.', + 'This option specifies the maximum length DHCP message that the\nclient is willing to accept.\n', }, { name: 'class_identifier', type: 'keyword', description: - "This option is used by DHCP clients to optionally identify the vendor type and configuration of a DHCP client. Vendors may choose to define specific vendor class identifiers to convey particular configuration or other identification information about a client. For example, the identifier may encode the client's hardware configuration. ", + "This option is used by DHCP clients to optionally identify the\nvendor type and configuration of a DHCP client. Vendors may\nchoose to define specific vendor class identifiers to convey\nparticular configuration or other identification information\nabout a client. For example, the identifier may encode the\nclient's hardware configuration.\n", }, { name: 'domain_name', type: 'keyword', description: - 'This option specifies the domain name that client should use when resolving hostnames via the Domain Name System.', + 'This option specifies the domain name that client should use\nwhen resolving hostnames via the Domain Name System.\n', }, { name: 'dns_servers', type: 'ip', description: - 'The domain name server option specifies a list of Domain Name System servers available to the client.', + 'The domain name server option specifies a list of Domain Name\nSystem servers available to the client.\n', }, { name: 'vendor_identifying_options', type: 'object', description: - 'A DHCP client may use this option to unambiguously identify the vendor that manufactured the hardware on which the client is running, the software in use, or an industry consortium to which the vendor belongs. This field is described in RFC 3925.', + 'A DHCP client may use this option to unambiguously identify the\nvendor that manufactured the hardware on which the client is\nrunning, the software in use, or an industry consortium to which\nthe vendor belongs. This field is described in RFC 3925.\n', }, { name: 'subnet_mask', type: 'ip', - description: 'The subnet mask that the client should use on the currnet network.', + description: + 'The subnet mask that the client should use on the currnet\nnetwork.\n', }, { name: 'utc_time_offset_sec', type: 'long', description: - "The time offset field specifies the offset of the client's subnet in seconds from Coordinated Universal Time (UTC). ", + "The time offset field specifies the offset of the client's\nsubnet in seconds from Coordinated Universal Time (UTC).\n", }, { name: 'router', type: 'ip', description: - "The router option specifies a list of IP addresses for routers on the client's subnet. ", + "The router option specifies a list of IP addresses for routers\non the client's subnet.\n", }, { name: 'time_servers', type: 'ip', description: - 'The time server option specifies a list of RFC 868 time servers available to the client.', + 'The time server option specifies a list of RFC 868 time servers\navailable to the client.\n', }, { name: 'ntp_servers', type: 'ip', description: - 'This option specifies a list of IP addresses indicating NTP servers available to the client.', + 'This option specifies a list of IP addresses indicating NTP\nservers available to the client.\n', }, { name: 'hostname', type: 'keyword', - description: 'This option specifies the name of the client.', + description: 'This option specifies the name of the client.\n', }, { name: 'ip_address_lease_time_sec', type: 'long', description: - 'This option is used in a client request (DHCPDISCOVER or DHCPREQUEST) to allow the client to request a lease time for the IP address. In a server reply (DHCPOFFER), a DHCP server uses this option to specify the lease time it is willing to offer.', + 'This option is used in a client request (DHCPDISCOVER or\nDHCPREQUEST) to allow the client to request a lease time for the\nIP address. In a server reply (DHCPOFFER), a DHCP server uses\nthis option to specify the lease time it is willing to offer.\n', }, { name: 'message', type: 'text', description: - 'This option is used by a DHCP server to provide an error message to a DHCP client in a DHCPNAK message in the event of a failure. A client may use this option in a DHCPDECLINE message to indicate the why the client declined the offered parameters.', + 'This option is used by a DHCP server to provide an error message\nto a DHCP client in a DHCPNAK message in the event of a failure.\nA client may use this option in a DHCPDECLINE message to\nindicate the why the client declined the offered parameters.\n', }, { name: 'renewal_time_sec', type: 'long', description: - 'This option specifies the time interval from address assignment until the client transitions to the RENEWING state.', + 'This option specifies the time interval from address assignment\nuntil the client transitions to the RENEWING state.\n', }, { name: 'rebinding_time_sec', type: 'long', description: - 'This option specifies the time interval from address assignment until the client transitions to the REBINDING state.', + 'This option specifies the time interval from address assignment\nuntil the client transitions to the REBINDING state.\n', }, { name: 'boot_file_name', type: 'keyword', description: - "This option is used to identify a bootfile when the 'file' field in the DHCP header has been used for DHCP options. ", + "This option is used to identify a bootfile when the 'file' field\nin the DHCP header has been used for DHCP options.\n", }, ], }, @@ -3451,129 +7001,64 @@ export const packetbeatSchema: Schema = [ name: 'dns', type: 'group', fields: [ - { - name: 'id', - type: 'long', - description: - 'The DNS packet identifier assigned by the program that generated the query. The identifier is copied to the response.', - }, - { - name: 'op_code', - description: - 'The DNS operation code that specifies the kind of query in the message. This value is set by the originator of a query and copied into the response.', - example: 'QUERY', - }, { name: 'flags.authoritative', type: 'boolean', description: - 'A DNS flag specifying that the responding server is an authority for the domain name used in the question.', + 'A DNS flag specifying that the responding server is an authority for the domain name used in the question.\n', }, { name: 'flags.recursion_available', type: 'boolean', description: - 'A DNS flag specifying whether recursive query support is available in the name server.', + 'A DNS flag specifying whether recursive query support is available in the name server.\n', }, { name: 'flags.recursion_desired', type: 'boolean', description: - 'A DNS flag specifying that the client directs the server to pursue a query recursively. Recursive query support is optional.', + 'A DNS flag specifying that the client directs the server to pursue a query recursively. Recursive query support is optional.\n', }, { name: 'flags.authentic_data', type: 'boolean', description: - 'A DNS flag specifying that the recursive server considers the response authentic.', + 'A DNS flag specifying that the recursive server considers the response authentic.\n', }, { name: 'flags.checking_disabled', type: 'boolean', description: - 'A DNS flag specifying that the client disables the server signature validation of the query.', + 'A DNS flag specifying that the client disables the server signature validation of the query.\n', }, { name: 'flags.truncated_response', type: 'boolean', description: - 'A DNS flag specifying that only the first 512 bytes of the reply were returned.', - }, - { - name: 'response_code', - description: 'The DNS status code.', - example: 'NOERROR', - }, - { - name: 'question.name', - description: - 'The domain name being queried. If the name field contains non-printable characters (below 32 or above 126), then those characters are represented as escaped base 10 integers (\\DDD). Back slashes and quotes are escaped. Tabs, carriage returns, and line feeds are converted to \\t, \\r, and respectively.', - example: 'www.google.com.', - }, - { - name: 'question.type', - description: 'The type of records being queried.', - example: 'AAAA', - }, - { - name: 'question.class', - description: 'The class of of records being queried.', - example: 'IN', + 'A DNS flag specifying that only the first 512 bytes of the reply were returned.\n', }, { name: 'question.etld_plus_one', description: - 'The effective top-level domain (eTLD) plus one more label. For example, the eTLD+1 for "foo.bar.golang.org." is "golang.org.". The data for determining the eTLD comes from an embedded copy of the data from http://publicsuffix.org.', + 'The effective top-level domain (eTLD) plus one more label.\nFor example, the eTLD+1 for "foo.bar.golang.org." is "golang.org.".\nThe data for determining the eTLD comes from an embedded copy of the\ndata from http://publicsuffix.org.', example: 'amazon.co.uk.', }, - { - name: 'answers', - type: 'object', - description: - 'An array containing a dictionary about each answer section returned by the server.', - }, { name: 'answers_count', type: 'long', - description: 'The number of resource records contained in the `dns.answers` field.', - }, - { - name: 'answers.name', - description: 'The domain name to which this resource record pertains.', - example: 'example.com.', - }, - { - name: 'answers.type', - description: 'The type of data contained in this resource record.', - example: 'MX', - }, - { - name: 'answers.class', - description: 'The class of DNS data contained in this resource record.', - example: 'IN', - }, - { - name: 'answers.ttl', - description: - 'The time interval in seconds that this resource record may be cached before it should be discarded. Zero values mean that the data should not be cached.', - type: 'long', - }, - { - name: 'answers.data', - description: - 'The data describing the resource. The meaning of this data depends on the type and class of the resource record.', + description: 'The number of resource records contained in the `dns.answers` field.\n', }, { name: 'authorities', type: 'object', description: - 'An array containing a dictionary for each authority section from the answer.', + 'An array containing a dictionary for each authority section from the answer.\n', }, { name: 'authorities_count', type: 'long', description: - 'The number of resource records contained in the `dns.authorities` field. The `dns.authorities` field may or may not be included depending on the configuration of Packetbeat.', + 'The number of resource records contained in the `dns.authorities` field. The `dns.authorities` field may or may not be included depending on the configuration of Packetbeat.\n', }, { name: 'authorities.name', @@ -3594,13 +7079,13 @@ export const packetbeatSchema: Schema = [ name: 'additionals', type: 'object', description: - 'An array containing a dictionary for each additional section from the answer.', + 'An array containing a dictionary for each additional section from the answer.\n', }, { name: 'additionals_count', type: 'long', description: - 'The number of resource records contained in the `dns.additionals` field. The `dns.additionals` field may or may not be included depending on the configuration of Packetbeat.', + 'The number of resource records contained in the `dns.additionals` field. The `dns.additionals` field may or may not be included depending on the configuration of Packetbeat.\n', }, { name: 'additionals.name', @@ -3620,13 +7105,13 @@ export const packetbeatSchema: Schema = [ { name: 'additionals.ttl', description: - 'The time interval in seconds that this resource record may be cached before it should be discarded. Zero values mean that the data should not be cached.', + 'The time interval in seconds that this resource record may be cached before it should be discarded. Zero values mean that the data should not be cached.\n', type: 'long', }, { name: 'additionals.data', description: - 'The data describing the resource. The meaning of this data depends on the type and class of the resource record.', + 'The data describing the resource. The meaning of this data depends on the type and class of the resource record.\n', }, { name: 'opt.version', @@ -3672,7 +7157,7 @@ export const packetbeatSchema: Schema = [ type: 'object', object_type: 'keyword', description: - 'A map containing the captured header fields from the request. Which headers to capture is configurable. If headers with the same header name are present in the message, they will be separated by commas.', + 'A map containing the captured header fields from the request. Which headers to capture is configurable. If headers with the same header name are present in the message, they will be separated by commas.\n', }, { name: 'params', @@ -3697,7 +7182,7 @@ export const packetbeatSchema: Schema = [ type: 'object', object_type: 'keyword', description: - 'A map containing the captured header fields from the response. Which headers to capture is configurable. If headers with the same header name are present in the message, they will be separated by commas.', + 'A map containing the captured header fields from the response. Which headers to capture is configurable. If headers with the same header name are present in the message, they will be separated by commas.\n', }, { name: 'code', @@ -3720,7 +7205,7 @@ export const packetbeatSchema: Schema = [ { key: 'icmp', title: 'ICMP', - description: 'ICMP specific event fields.', + description: 'ICMP specific event fields.\n', fields: [ { name: 'icmp', @@ -3778,232 +7263,232 @@ export const packetbeatSchema: Schema = [ name: 'protocol_type', type: 'keyword', description: - 'The memcache protocol implementation. The value can be "binary" for binary-based, "text" for text-based, or "unknown" for an unknown memcache protocol type.', + 'The memcache protocol implementation. The value can be "binary" for binary-based, "text" for text-based, or "unknown" for an unknown memcache protocol type.\n', }, { name: 'request.line', type: 'keyword', - description: 'The raw command line for unknown commands ONLY.', + description: 'The raw command line for unknown commands ONLY.\n', }, { name: 'request.command', type: 'keyword', description: - 'The memcache command being requested in the memcache text protocol. For example "set" or "get". The binary protocol opcodes are translated into memcache text protocol commands.', + 'The memcache command being requested in the memcache text protocol. For example "set" or "get". The binary protocol opcodes are translated into memcache text protocol commands.\n', }, { name: 'response.command', type: 'keyword', description: - 'Either the text based protocol response message type or the name of the originating request if binary protocol is used.', + 'Either the text based protocol response message type or the name of the originating request if binary protocol is used.\n', }, { name: 'request.type', type: 'keyword', description: - 'The memcache command classification. This value can be "UNKNOWN", "Load", "Store", "Delete", "Counter", "Info", "SlabCtrl", "LRUCrawler", "Stats", "Success", "Fail", or "Auth".', + 'The memcache command classification. This value can be "UNKNOWN", "Load", "Store", "Delete", "Counter", "Info", "SlabCtrl", "LRUCrawler", "Stats", "Success", "Fail", or "Auth".\n', }, { name: 'response.type', type: 'keyword', description: - 'The memcache command classification. This value can be "UNKNOWN", "Load", "Store", "Delete", "Counter", "Info", "SlabCtrl", "LRUCrawler", "Stats", "Success", "Fail", or "Auth". The text based protocol will employ any of these, whereas the binary based protocol will mirror the request commands only (see `memcache.response.status` for binary protocol).', + 'The memcache command classification. This value can be "UNKNOWN", "Load", "Store", "Delete", "Counter", "Info", "SlabCtrl", "LRUCrawler", "Stats", "Success", "Fail", or "Auth". The text based protocol will employ any of these, whereas the binary based protocol will mirror the request commands only (see `memcache.response.status` for binary protocol).\n', }, { name: 'response.error_msg', type: 'keyword', description: - 'The optional error message in the memcache response (text based protocol only).', + 'The optional error message in the memcache response (text based protocol only).\n', }, { name: 'request.opcode', type: 'keyword', - description: 'The binary protocol message opcode name.', + description: 'The binary protocol message opcode name.\n', }, { name: 'response.opcode', type: 'keyword', - description: 'The binary protocol message opcode name.', + description: 'The binary protocol message opcode name.\n', }, { name: 'request.opcode_value', type: 'long', - description: 'The binary protocol message opcode value.', + description: 'The binary protocol message opcode value.\n', }, { name: 'response.opcode_value', type: 'long', - description: 'The binary protocol message opcode value.', + description: 'The binary protocol message opcode value.\n', }, { name: 'request.opaque', type: 'long', description: - 'The binary protocol opaque header value used for correlating request with response messages.', + 'The binary protocol opaque header value used for correlating request with response messages.\n', }, { name: 'response.opaque', type: 'long', description: - 'The binary protocol opaque header value used for correlating request with response messages.', + 'The binary protocol opaque header value used for correlating request with response messages.\n', }, { name: 'request.vbucket', type: 'long', - description: 'The vbucket index sent in the binary message.', + description: 'The vbucket index sent in the binary message.\n', }, { name: 'response.status', type: 'keyword', description: - 'The textual representation of the response error code (binary protocol only).', + 'The textual representation of the response error code (binary protocol only).\n', }, { name: 'response.status_code', type: 'long', - description: 'The status code value returned in the response (binary protocol only).', + description: 'The status code value returned in the response (binary protocol only).\n', }, { name: 'request.keys', type: 'array', - description: 'The list of keys sent in the store or load commands.', + description: 'The list of keys sent in the store or load commands.\n', }, { name: 'response.keys', type: 'array', - description: 'The list of keys returned for the load command (if present).', + description: 'The list of keys returned for the load command (if present).\n', }, { name: 'request.count_values', type: 'long', description: - 'The number of values found in the memcache request message. If the command does not send any data, this field is missing.', + 'The number of values found in the memcache request message. If the command does not send any data, this field is missing.\n', }, { name: 'response.count_values', type: 'long', description: - 'The number of values found in the memcache response message. If the command does not send any data, this field is missing.', + 'The number of values found in the memcache response message. If the command does not send any data, this field is missing.\n', }, { name: 'request.values', type: 'array', - description: 'The list of base64 encoded values sent with the request (if present).', + description: 'The list of base64 encoded values sent with the request (if present).\n', }, { name: 'response.values', type: 'array', - description: 'The list of base64 encoded values sent with the response (if present).', + description: 'The list of base64 encoded values sent with the response (if present).\n', }, { name: 'request.bytes', type: 'long', format: 'bytes', - description: 'The byte count of the values being transferred.', + description: 'The byte count of the values being transferred.\n', }, { name: 'response.bytes', type: 'long', format: 'bytes', - description: 'The byte count of the values being transferred.', + description: 'The byte count of the values being transferred.\n', }, { name: 'request.delta', type: 'long', - description: 'The counter increment/decrement delta value.', + description: 'The counter increment/decrement delta value.\n', }, { name: 'request.initial', type: 'long', description: - 'The counter increment/decrement initial value parameter (binary protocol only).', + 'The counter increment/decrement initial value parameter (binary protocol only).\n', }, { name: 'request.verbosity', type: 'long', - description: 'The value of the memcache "verbosity" command.', + description: 'The value of the memcache "verbosity" command.\n', }, { name: 'request.raw_args', type: 'keyword', description: - 'The text protocol raw arguments for the "stats ..." and "lru crawl ..." commands.', + 'The text protocol raw arguments for the "stats ..." and "lru crawl ..." commands.\n', }, { name: 'request.source_class', type: 'long', - description: "The source class id in 'slab reassign' command. ", + description: "The source class id in 'slab reassign' command.\n", }, { name: 'request.dest_class', type: 'long', - description: "The destination class id in 'slab reassign' command. ", + description: "The destination class id in 'slab reassign' command.\n", }, { name: 'request.automove', type: 'keyword', description: - 'The automove mode in the \'slab automove\' command expressed as a string. This value can be "standby"(=0), "slow"(=1), "aggressive"(=2), or the raw value if the value is unknown.', + 'The automove mode in the \'slab automove\' command expressed as a string. This value can be "standby"(=0), "slow"(=1), "aggressive"(=2), or the raw value if the value is unknown.\n', }, { name: 'request.flags', type: 'long', - description: 'The memcache command flags sent in the request (if present).', + description: 'The memcache command flags sent in the request (if present).\n', }, { name: 'response.flags', type: 'long', - description: 'The memcache message flags sent in the response (if present).', + description: 'The memcache message flags sent in the response (if present).\n', }, { name: 'request.exptime', type: 'long', description: - 'The data expiry time in seconds sent with the memcache command (if present). If the value is <30 days, the expiry time is relative to "now", or else it is an absolute Unix time in seconds (32-bit).', + 'The data expiry time in seconds sent with the memcache command (if present). If the value is <30 days, the expiry time is relative to "now", or else it is an absolute Unix time in seconds (32-bit).\n', }, { name: 'request.sleep_us', type: 'long', - description: "The sleep setting in microseconds for the 'lru_crawler sleep' command. ", + description: "The sleep setting in microseconds for the 'lru_crawler sleep' command.\n", }, { name: 'response.value', type: 'long', - description: 'The counter value returned by a counter operation.', + description: 'The counter value returned by a counter operation.\n', }, { name: 'request.noreply', type: 'boolean', description: - 'Set to true if noreply was set in the request. The `memcache.response` field will be missing.', + 'Set to true if noreply was set in the request. The `memcache.response` field will be missing.\n', }, { name: 'request.quiet', type: 'boolean', description: - 'Set to true if the binary protocol message is to be treated as a quiet message.', + 'Set to true if the binary protocol message is to be treated as a quiet message.\n', }, { name: 'request.cas_unique', type: 'long', - description: 'The CAS (compare-and-swap) identifier if present.', + description: 'The CAS (compare-and-swap) identifier if present.\n', }, { name: 'response.cas_unique', type: 'long', description: - 'The CAS (compare-and-swap) identifier to be used with CAS-based updates (if present).', + 'The CAS (compare-and-swap) identifier to be used with CAS-based updates (if present).\n', }, { name: 'response.stats', type: 'array', description: - 'The list of statistic values returned. Each entry is a dictionary with the fields "name" and "value".', + 'The list of statistic values returned. Each entry is a dictionary with the fields "name" and "value".\n', }, { name: 'response.version', type: 'keyword', - description: 'The returned memcache version string.', + description: 'The returned memcache version string.\n', }, ], }, @@ -4013,7 +7498,7 @@ export const packetbeatSchema: Schema = [ key: 'mongodb', title: 'MongoDb', description: - 'MongoDB-specific event fields. These fields mirror closely the fields for the MongoDB wire protocol. The higher level fields (for example, `query` and `resource`) apply to MongoDB events as well.', + 'MongoDB-specific event fields. These fields mirror closely the fields for the MongoDB wire protocol. The higher level fields (for example, `query` and `resource`) apply to MongoDB events as well.\n', fields: [ { name: 'mongodb', @@ -4022,57 +7507,57 @@ export const packetbeatSchema: Schema = [ { name: 'error', description: - 'If the MongoDB request has resulted in an error, this field contains the error message returned by the server.', + 'If the MongoDB request has resulted in an error, this field contains the error message returned by the server.\n', }, { name: 'fullCollectionName', description: - 'The full collection name. The full collection name is the concatenation of the database name with the collection name, using a dot (.) for the concatenation. For example, for the database foo and the collection bar, the full collection name is foo.bar.', + 'The full collection name. The full collection name is the concatenation of the database name with the collection name, using a dot (.) for the concatenation. For example, for the database foo and the collection bar, the full collection name is foo.bar.\n', }, { name: 'numberToSkip', type: 'long', description: - 'Sets the number of documents to omit - starting from the first document in the resulting dataset - when returning the result of the query.', + 'Sets the number of documents to omit - starting from the first document in the resulting dataset - when returning the result of the query.\n', }, { name: 'numberToReturn', type: 'long', - description: 'The requested maximum number of documents to be returned.', + description: 'The requested maximum number of documents to be returned.\n', }, { name: 'numberReturned', type: 'long', - description: 'The number of documents in the reply.', + description: 'The number of documents in the reply.\n', }, { name: 'startingFrom', - description: 'Where in the cursor this reply is starting.', + description: 'Where in the cursor this reply is starting.\n', }, { name: 'query', description: - 'A JSON document that represents the query. The query will contain one or more elements, all of which must match for a document to be included in the result set. Possible elements include $query, $orderby, $hint, $explain, and $snapshot.', + 'A JSON document that represents the query. The query will contain one or more elements, all of which must match for a document to be included in the result set. Possible elements include $query, $orderby, $hint, $explain, and $snapshot.\n', }, { name: 'returnFieldsSelector', description: - 'A JSON document that limits the fields in the returned documents. The returnFieldsSelector contains one or more elements, each of which is the name of a field that should be returned, and the integer value 1.', + 'A JSON document that limits the fields in the returned documents. The returnFieldsSelector contains one or more elements, each of which is the name of a field that should be returned, and the integer value 1.\n', }, { name: 'selector', description: - 'A BSON document that specifies the query for selecting the document to update or delete.', + 'A BSON document that specifies the query for selecting the document to update or delete.\n', }, { name: 'update', description: - 'A BSON document that specifies the update to be performed. For information on specifying updates, see the Update Operations documentation from the MongoDB Manual.', + 'A BSON document that specifies the update to be performed. For information on specifying updates, see the Update Operations documentation from the MongoDB Manual.\n', }, { name: 'cursorId', description: - 'The cursor identifier returned in the OP_REPLY. This must be the value that was returned from the database.', + 'The cursor identifier returned in the OP_REPLY. This must be the value that was returned from the database.\n', }, ], }, @@ -4081,7 +7566,7 @@ export const packetbeatSchema: Schema = [ { key: 'mysql', title: 'MySQL', - description: 'MySQL-specific event fields.', + description: 'MySQL-specific event fields.\n', fields: [ { name: 'mysql', @@ -4091,35 +7576,35 @@ export const packetbeatSchema: Schema = [ name: 'affected_rows', type: 'long', description: - 'If the MySQL command is successful, this field contains the affected number of rows of the last statement.', + 'If the MySQL command is successful, this field contains the affected number of rows of the last statement.\n', }, { name: 'insert_id', description: - 'If the INSERT query is successful, this field contains the id of the newly inserted row.', + 'If the INSERT query is successful, this field contains the id of the newly inserted row.\n', }, { name: 'num_fields', description: - 'If the SELECT query is successful, this field is set to the number of fields returned.', + 'If the SELECT query is successful, this field is set to the number of fields returned.\n', }, { name: 'num_rows', description: - 'If the SELECT query is successful, this field is set to the number of rows returned.', + 'If the SELECT query is successful, this field is set to the number of rows returned.\n', }, { name: 'query', - description: "The row mysql query as read from the transaction's request. ", + description: "The row mysql query as read from the transaction's request.\n", }, { name: 'error_code', type: 'long', - description: 'The error code returned by MySQL.', + description: 'The error code returned by MySQL.\n', }, { name: 'error_message', - description: 'The error info message returned by MySQL.', + description: 'The error info message returned by MySQL.\n', }, ], }, @@ -4150,7 +7635,7 @@ export const packetbeatSchema: Schema = [ }, { name: 'opcode', - description: 'NFS operation name, or main operation name, in case of COMPOUND calls.', + description: 'NFS operation name, or main operation name, in case of COMPOUND calls.\n', }, { name: 'status', @@ -4219,7 +7704,7 @@ export const packetbeatSchema: Schema = [ { key: 'pgsql', title: 'PostgreSQL', - description: 'PostgreSQL-specific event fields.', + description: 'PostgreSQL-specific event fields.\n', fields: [ { name: 'pgsql', @@ -4242,12 +7727,12 @@ export const packetbeatSchema: Schema = [ { name: 'num_fields', description: - 'If the SELECT query if successful, this field is set to the number of fields returned.', + 'If the SELECT query if successful, this field is set to the number of fields returned.\n', }, { name: 'num_rows', description: - 'If the SELECT query if successful, this field is set to the number of rows returned.', + 'If the SELECT query if successful, this field is set to the number of rows returned.\n', }, ], }, @@ -4256,7 +7741,7 @@ export const packetbeatSchema: Schema = [ { key: 'redis', title: 'Redis', - description: 'Redis-specific event fields.', + description: 'Redis-specific event fields.\n', fields: [ { name: 'redis', @@ -4264,12 +7749,12 @@ export const packetbeatSchema: Schema = [ fields: [ { name: 'return_value', - description: 'The return value of the Redis command in a human readable format.', + description: 'The return value of the Redis command in a human readable format.\n', }, { name: 'error', description: - 'If the Redis command has resulted in an error, this field contains the error message returned by the Redis server.', + 'If the Redis command has resulted in an error, this field contains the error message returned by the Redis server.\n', }, ], }, @@ -4278,7 +7763,7 @@ export const packetbeatSchema: Schema = [ { key: 'thrift', title: 'Thrift-RPC', - description: 'Thrift-RPC specific event fields.', + description: 'Thrift-RPC specific event fields.\n', fields: [ { name: 'thrift', @@ -4287,521 +7772,785 @@ export const packetbeatSchema: Schema = [ { name: 'params', description: - 'The RPC method call parameters in a human readable format. If the IDL files are available, the parameters use names whenever possible. Otherwise, the IDs from the message are used.', + 'The RPC method call parameters in a human readable format. If the IDL files are available, the parameters use names whenever possible. Otherwise, the IDs from the message are used.\n', }, { name: 'service', - description: 'The name of the Thrift-RPC service as defined in the IDL files.', + description: 'The name of the Thrift-RPC service as defined in the IDL files.\n', }, { name: 'return_value', description: - 'The value returned by the Thrift-RPC call. This is encoded in a human readable format.', + 'The value returned by the Thrift-RPC call. This is encoded in a human readable format.\n', }, { name: 'exceptions', description: - 'If the call resulted in exceptions, this field contains the exceptions in a human readable format.', + 'If the call resulted in exceptions, this field contains the exceptions in a human readable format.\n', }, ], }, ], - }, - { - key: 'tls', - title: 'TLS', - description: 'TLS-specific event fields.', - fields: [ - { - name: 'tls', - type: 'group', - fields: [ - { - name: 'version', - type: 'keyword', - description: 'The version of the TLS protocol used.', - example: 'TLS 1.3', - }, - { - name: 'handshake_completed', - type: 'boolean', - description: - 'Whether the TLS negotiation has been successful and the session has transitioned to encrypted mode.', - }, - { - name: 'resumed', - type: 'boolean', - description: 'If the TLS session has been resumed from a previous session.', - }, - { - name: 'resumption_method', - type: 'keyword', - description: - 'If the session has been resumed, the underlying method used. One of "id" for TLS session ID or "ticket" for TLS ticket extension.', - }, - { - name: 'client_certificate_requested', - type: 'boolean', - description: - 'Whether the server has requested the client to authenticate itself using a client certificate.', - }, - { - name: 'client_hello', - type: 'group', - fields: [ - { - name: 'version', - type: 'keyword', - description: - 'The version of the TLS protocol by which the client wishes to communicate during this session.', - }, - { - name: 'supported_ciphers', - type: 'array', - description: - 'List of ciphers the client is willing to use for this session. See https://www.iana.org/assignments/tls-parameters/tls-parameters.xhtml#tls-parameters-4', - }, - { - name: 'supported_compression_methods', - type: 'array', - description: - 'The list of compression methods the client supports. See https://www.iana.org/assignments/comp-meth-ids/comp-meth-ids.xhtml', - }, - { - name: 'extensions', - type: 'group', - description: 'The hello extensions provided by the client.', - fields: [ - { - name: 'server_name_indication', - type: 'keyword', - description: 'List of hostnames', - }, - { - name: 'application_layer_protocol_negotiation', - type: 'keyword', - description: - 'List of application-layer protocols the client is willing to use.', - }, - { - name: 'session_ticket', - type: 'keyword', - description: - 'Length of the session ticket, if provided, or an empty string to advertise support for tickets.', - }, - { - name: 'supported_versions', - type: 'keyword', - description: 'List of TLS versions that the client is willing to use.', - }, - { - name: 'supported_groups', - type: 'keyword', - description: - 'List of Elliptic Curve Cryptography (ECC) curve groups supported by the client.', - }, - { - name: 'signature_algorithms', - type: 'keyword', - description: - 'List of signature algorithms that may be use in digital signatures.', - }, - { - name: 'ec_points_formats', - type: 'keyword', - description: - 'List of Elliptic Curve (EC) point formats. Indicates the set of point formats that the client can parse.', - }, - { - name: '_unparsed_', - type: 'keyword', - description: 'List of extensions that were left unparsed by Packetbeat.', - }, - ], - }, - ], - }, + }, + { + key: 'tls_detailed', + title: 'Detailed TLS', + description: 'Detailed TLS-specific event fields.\n', + fields: [ + { + name: 'tls', + type: 'group', + fields: [ { - name: 'server_hello', + name: 'detailed', type: 'group', + default_fields: false, fields: [ { name: 'version', type: 'keyword', - description: - 'The version of the TLS protocol that is used for this session. It is the highest version supported by the server not exceeding the version requested in the client hello.', - }, - { - name: 'selected_cipher', - type: 'keyword', - description: - 'The cipher suite selected by the server from the list provided by in the client hello.', + description: 'The version of the TLS protocol used.\n', + example: 'TLS 1.3', }, { - name: 'selected_compression_method', + name: 'resumption_method', type: 'keyword', description: - 'The compression method selected by the server from the list provided in the client hello.', + 'If the session has been resumed, the underlying method used. One of "id" for TLS session ID or "ticket" for TLS ticket extension.\n', }, { - name: 'session_id', - type: 'keyword', + name: 'client_certificate_requested', + type: 'boolean', description: - 'Unique number to identify the session for the corresponding connection with the client.', + 'Whether the server has requested the client to authenticate itself using a client certificate.\n', }, { - name: 'extensions', + name: 'client_hello', type: 'group', - description: 'The hello extensions provided by the server.', fields: [ { - name: 'application_layer_protocol_negotiation', - type: 'array', - description: 'Negotiated application layer protocol', - }, - { - name: 'session_ticket', + name: 'version', type: 'keyword', description: - 'Used to announce that a session ticket will be provided by the server. Always an empty string.', + 'The version of the TLS protocol by which the client wishes to communicate during this session.\n', }, { - name: 'supported_versions', + name: 'session_id', type: 'keyword', - description: 'Negotiated TLS version to be used.', + description: + 'Unique number to identify the session for the corresponding connection with the client.\n', }, { - name: 'ec_points_formats', + name: 'supported_compression_methods', type: 'keyword', description: - 'List of Elliptic Curve (EC) point formats. Indicates the set of point formats that the server can parse.', + 'The list of compression methods the client supports. See https://www.iana.org/assignments/comp-meth-ids/comp-meth-ids.xhtml\n', }, { - name: '_unparsed_', - type: 'keyword', - description: 'List of extensions that were left unparsed by Packetbeat.', + name: 'extensions', + type: 'group', + description: 'The hello extensions provided by the client.', + fields: [ + { + name: 'server_name_indication', + type: 'keyword', + description: 'List of hostnames', + }, + { + name: 'application_layer_protocol_negotiation', + type: 'keyword', + description: + 'List of application-layer protocols the client is willing to use.\n', + }, + { + name: 'session_ticket', + type: 'keyword', + description: + 'Length of the session ticket, if provided, or an empty string to advertise support for tickets.\n', + }, + { + name: 'supported_versions', + type: 'keyword', + description: 'List of TLS versions that the client is willing to use.\n', + }, + { + name: 'supported_groups', + type: 'keyword', + description: + 'List of Elliptic Curve Cryptography (ECC) curve groups supported by the client.\n', + }, + { + name: 'signature_algorithms', + type: 'keyword', + description: + 'List of signature algorithms that may be use in digital signatures.\n', + }, + { + name: 'ec_points_formats', + type: 'keyword', + description: + 'List of Elliptic Curve (EC) point formats. Indicates the set of point formats that the client can parse.\n', + }, + { + name: '_unparsed_', + type: 'keyword', + description: 'List of extensions that were left unparsed by Packetbeat.\n', + }, + ], }, ], }, - ], - }, - { - name: 'client_certificate', - type: 'group', - description: 'Certificate provided by the client for authentication.', - fields: [ - { - name: 'version', - type: 'long', - description: 'X509 format version.', - }, - { - name: 'serial_number', - type: 'keyword', - description: "The certificate's serial number.", - }, - { - name: 'not_before', - type: 'date', - description: 'Date before which the certificate is not valid.', - }, { - name: 'not_after', - type: 'date', - description: 'Date after which the certificate expires.', - }, - { - name: 'public_key_algorithm', - type: 'keyword', - description: - "The algorithm used for this certificate's public key. One of RSA, DSA or ECDSA. ", - }, - { - name: 'public_key_size', - type: 'long', - description: 'Size of the public key.', - }, - { - name: 'signature_algorithm', - type: 'keyword', - description: "The algorithm used for the certificate's signature. ", - }, - { - name: 'alternative_names', - type: 'array', - description: 'Subject Alternative Names for this certificate.', - }, - { - name: 'raw', - type: 'keyword', - description: 'The raw certificate in PEM format.', - }, - { - name: 'subject', + name: 'server_hello', type: 'group', - description: 'Subject represented by this certificate.', fields: [ { - name: 'country', - type: 'keyword', - description: 'Country code.', - }, - { - name: 'organization', + name: 'version', type: 'keyword', - description: 'Organization name.', + description: + 'The version of the TLS protocol that is used for this session. It is the highest version supported by the server not exceeding the version requested in the client hello.\n', }, { - name: 'organizational_unit', + name: 'selected_compression_method', type: 'keyword', - description: 'Unit within organization.', + description: + 'The compression method selected by the server from the list provided in the client hello.\n', }, { - name: 'province', + name: 'session_id', type: 'keyword', - description: 'Province or region within country.', + description: + 'Unique number to identify the session for the corresponding connection with the client.\n', }, { - name: 'common_name', - type: 'keyword', - description: 'Name or host name identified by the certificate.', + name: 'extensions', + type: 'group', + description: 'The hello extensions provided by the server.', + fields: [ + { + name: 'application_layer_protocol_negotiation', + type: 'keyword', + description: 'Negotiated application layer protocol', + }, + { + name: 'session_ticket', + type: 'keyword', + description: + 'Used to announce that a session ticket will be provided by the server. Always an empty string.\n', + }, + { + name: 'supported_versions', + type: 'keyword', + description: 'Negotiated TLS version to be used.\n', + }, + { + name: 'ec_points_formats', + type: 'keyword', + description: + 'List of Elliptic Curve (EC) point formats. Indicates the set of point formats that the server can parse.\n', + }, + { + name: '_unparsed_', + type: 'keyword', + description: 'List of extensions that were left unparsed by Packetbeat.\n', + }, + ], }, ], }, { - name: 'issuer', + name: 'client_certificate', type: 'group', - description: 'Entity that issued and signed this certificate.', + description: 'Certificate provided by the client for authentication.', fields: [ { - name: 'country', - type: 'keyword', - description: 'Country code.', + name: 'version', + type: 'long', + description: 'X509 format version.', }, { - name: 'organization', + name: 'serial_number', type: 'keyword', - description: 'Organization name.', + description: "The certificate's serial number.", }, { - name: 'organizational_unit', - type: 'keyword', - description: 'Unit within organization.', + name: 'not_before', + type: 'date', + description: 'Date before which the certificate is not valid.', }, { - name: 'province', - type: 'keyword', - description: 'Province or region within country.', + name: 'not_after', + type: 'date', + description: 'Date after which the certificate expires.', }, { - name: 'common_name', + name: 'public_key_algorithm', type: 'keyword', - description: 'Name or host name identified by the certificate.', + description: + "The algorithm used for this certificate's public key. One of RSA, DSA or ECDSA.\n", }, - ], - }, - { - name: 'fingerprint', - type: 'group', - fields: [ { - name: 'md5', - type: 'keyword', - description: "Certificate's MD5 fingerprint.", + name: 'public_key_size', + type: 'long', + description: 'Size of the public key.', }, { - name: 'sha1', + name: 'signature_algorithm', type: 'keyword', - description: "Certificate's SHA-1 fingerprint.", + description: "The algorithm used for the certificate's signature.\n", }, { - name: 'sha256', + name: 'alternative_names', type: 'keyword', - description: "Certificate's SHA-256 fingerprint.", + description: 'Subject Alternative Names for this certificate.', + }, + { + name: 'subject', + type: 'group', + description: 'Subject represented by this certificate.', + fields: [ + { + name: 'country', + type: 'keyword', + description: 'Country code.', + }, + { + name: 'organization', + type: 'keyword', + description: 'Organization name.', + }, + { + name: 'organizational_unit', + type: 'keyword', + description: 'Unit within organization.', + }, + { + name: 'province', + type: 'keyword', + description: 'Province or region within country.', + }, + { + name: 'common_name', + type: 'keyword', + description: 'Name or host name identified by the certificate.', + }, + { + name: 'locality', + type: 'keyword', + description: 'Locality.', + }, + ], + }, + { + name: 'issuer', + type: 'group', + description: 'Entity that issued and signed this certificate.', + fields: [ + { + name: 'country', + type: 'keyword', + description: 'Country code.', + }, + { + name: 'organization', + type: 'keyword', + description: 'Organization name.', + }, + { + name: 'organizational_unit', + type: 'keyword', + description: 'Unit within organization.', + }, + { + name: 'province', + type: 'keyword', + description: 'Province or region within country.', + }, + { + name: 'common_name', + type: 'keyword', + description: 'Name or host name identified by the certificate.', + }, + { + name: 'locality', + type: 'keyword', + description: 'Locality.', + }, + ], }, ], }, - ], - }, - { - name: 'server_certificate', - type: 'group', - description: 'Certificate provided by the server for authentication.', - fields: [ - { - name: 'version', - type: 'long', - description: 'X509 format version.', - }, - { - name: 'serial_number', - type: 'keyword', - description: "The certificate's serial number.", - }, - { - name: 'not_before', - type: 'date', - description: 'Date before which the certificate is not valid.', - }, - { - name: 'not_after', - type: 'date', - description: 'Date after which the certificate expires.', - }, - { - name: 'public_key_algorithm', - type: 'keyword', - description: - "The algorithm used for this certificate's public key. One of RSA, DSA or ECDSA. ", - }, - { - name: 'public_key_size', - type: 'long', - description: 'Size of the public key.', - }, { - name: 'signature_algorithm', - type: 'keyword', - description: "The algorithm used for the certificate's signature. ", - }, - { - name: 'alternative_names', - type: 'array', - description: 'Subject Alternative Names for this certificate.', - }, - { - name: 'raw', - type: 'keyword', - description: 'The raw certificate in PEM format.', - }, - { - name: 'subject', + name: 'server_certificate', type: 'group', - description: 'Subject represented by this certificate.', + description: 'Certificate provided by the server for authentication.', fields: [ { - name: 'country', - type: 'keyword', - description: 'Country code.', + name: 'version', + type: 'long', + description: 'X509 format version.', }, { - name: 'organization', + name: 'serial_number', type: 'keyword', - description: 'Organization name.', + description: "The certificate's serial number.", + }, + { + name: 'not_before', + type: 'date', + description: 'Date before which the certificate is not valid.', + }, + { + name: 'not_after', + type: 'date', + description: 'Date after which the certificate expires.', }, { - name: 'organizational_unit', + name: 'public_key_algorithm', type: 'keyword', - description: 'Unit within organization.', + description: + "The algorithm used for this certificate's public key. One of RSA, DSA or ECDSA.\n", + }, + { + name: 'public_key_size', + type: 'long', + description: 'Size of the public key.', }, { - name: 'province', + name: 'signature_algorithm', type: 'keyword', - description: 'Province or region within country.', + description: "The algorithm used for the certificate's signature.\n", }, { - name: 'common_name', + name: 'alternative_names', type: 'keyword', - description: 'Name or host name identified by the certificate.', + description: 'Subject Alternative Names for this certificate.', + }, + { + name: 'subject', + type: 'group', + description: 'Subject represented by this certificate.', + fields: [ + { + name: 'country', + type: 'keyword', + description: 'Country code.', + }, + { + name: 'organization', + type: 'keyword', + description: 'Organization name.', + }, + { + name: 'organizational_unit', + type: 'keyword', + description: 'Unit within organization.', + }, + { + name: 'province', + type: 'keyword', + description: 'Province or region within country.', + }, + { + name: 'common_name', + type: 'keyword', + description: 'Name or host name identified by the certificate.', + }, + { + name: 'locality', + type: 'keyword', + description: 'Locality.', + }, + ], + }, + { + name: 'issuer', + type: 'group', + description: 'Entity that issued and signed this certificate.', + fields: [ + { + name: 'country', + type: 'keyword', + description: 'Country code.', + }, + { + name: 'organization', + type: 'keyword', + description: 'Organization name.', + }, + { + name: 'organizational_unit', + type: 'keyword', + description: 'Unit within organization.', + }, + { + name: 'province', + type: 'keyword', + description: 'Province or region within country.', + }, + { + name: 'common_name', + type: 'keyword', + description: 'Name or host name identified by the certificate.', + }, + { + name: 'locality', + type: 'keyword', + description: 'Locality.', + }, + ], }, ], }, { - name: 'issuer', - type: 'group', - description: 'Entity that issued and signed this certificate.', - fields: [ - { - name: 'country', - type: 'keyword', - description: 'Country code.', - }, - { - name: 'organization', - type: 'keyword', - description: 'Organization name.', - }, - { - name: 'organizational_unit', - type: 'keyword', - description: 'Unit within organization.', - }, - { - name: 'province', - type: 'keyword', - description: 'Province or region within country.', - }, - { - name: 'common_name', - type: 'keyword', - description: 'Name or host name identified by the certificate.', - }, - ], + name: 'server_certificate_chain', + type: 'array', + description: 'Chain of trust for the server certificate.', }, { - name: 'fingerprint', - type: 'group', - fields: [ - { - name: 'md5', - type: 'keyword', - description: "Certificate's MD5 fingerprint.", - }, - { - name: 'sha1', - type: 'keyword', - description: "Certificate's SHA-1 fingerprint.", - }, - { - name: 'sha256', - type: 'keyword', - description: "Certificate's SHA-256 fingerprint.", - }, - ], + name: 'client_certificate_chain', + type: 'array', + description: 'Chain of trust for the client certificate.', }, - ], - }, - { - name: 'server_certificate_chain', - type: 'array', - description: 'Chain of trust for the server certificate.', - }, - { - name: 'client_certificate_chain', - type: 'array', - description: 'Chain of trust for the client certificate.', - }, - { - name: 'alert_types', - type: 'keyword', - description: 'An array containing the TLS alert type for every alert received.', - }, - { - name: 'fingerprints', - type: 'group', - description: 'Fingerprints for this TLS session.', - fields: [ { - name: 'ja3', - type: 'group', - description: 'JA3 TLS client fingerprint', - fields: [ - { - name: 'hash', - type: 'keyword', - description: 'The JA3 fingerprint hash for the client side.', - }, - { - name: 'str', - type: 'keyword', - description: 'The JA3 string used to calculate the hash.', - }, - ], + name: 'alert_types', + type: 'keyword', + description: 'An array containing the TLS alert type for every alert received.\n', }, ], }, ], }, + { + name: 'tls.handshake_completed', + type: 'alias', + path: 'tls.established', + }, + { + name: 'tls.client_hello.supported_ciphers', + type: 'alias', + path: 'tls.client.supported_ciphers', + }, + { + name: 'tls.server_hello.selected_cipher', + type: 'alias', + path: 'tls.cipher', + }, + { + name: 'tls.fingerprints.ja3', + type: 'alias', + path: 'tls.client.ja3', + }, + { + name: 'tls.resumption_method', + type: 'alias', + path: 'tls.detailed.resumption_method', + }, + { + name: 'tls.client_certificate_requested', + type: 'alias', + path: 'tls.detailed.client_certificate_requested', + }, + { + name: 'tls.client_hello.version', + type: 'alias', + path: 'tls.detailed.client_hello.version', + }, + { + name: 'tls.client_hello.session_id', + type: 'alias', + path: 'tls.detailed.client_hello.session_id', + }, + { + name: 'tls.client_hello.supported_compression_methods', + type: 'alias', + path: 'tls.detailed.client_hello.supported_compression_methods', + }, + { + name: 'tls.client_hello.extensions.server_name_indication', + type: 'alias', + path: 'tls.detailed.client_hello.extensions.server_name_indication', + }, + { + name: 'tls.client_hello.extensions.application_layer_protocol_negotiation', + type: 'alias', + path: 'tls.detailed.client_hello.extensions.application_layer_protocol_negotiation', + }, + { + name: 'tls.client_hello.extensions.session_ticket', + type: 'alias', + path: 'tls.detailed.client_hello.extensions.session_ticket', + }, + { + name: 'tls.client_hello.extensions.supported_versions', + type: 'alias', + path: 'tls.detailed.client_hello.extensions.supported_versions', + }, + { + name: 'tls.client_hello.extensions.supported_groups', + type: 'alias', + path: 'tls.detailed.client_hello.extensions.supported_groups', + }, + { + name: 'tls.client_hello.extensions.signature_algorithms', + type: 'alias', + path: 'tls.detailed.client_hello.extensions.signature_algorithms', + }, + { + name: 'tls.client_hello.extensions.ec_points_formats', + type: 'alias', + path: 'tls.detailed.client_hello.extensions.ec_points_formats', + }, + { + name: 'tls.client_hello.extensions._unparsed_', + type: 'alias', + path: 'tls.detailed.client_hello.extensions._unparsed_', + }, + { + name: 'tls.server_hello.version', + type: 'alias', + path: 'tls.detailed.server_hello.version', + }, + { + name: 'tls.server_hello.selected_compression_method', + type: 'alias', + path: 'tls.detailed.server_hello.selected_compression_method', + }, + { + name: 'tls.server_hello.session_id', + type: 'alias', + path: 'tls.detailed.server_hello.session_id', + }, + { + name: 'tls.server_hello.extensions.application_layer_protocol_negotiation', + type: 'alias', + path: 'tls.detailed.server_hello.extensions.application_layer_protocol_negotiation', + }, + { + name: 'tls.server_hello.extensions.session_ticket', + type: 'alias', + path: 'tls.detailed.server_hello.extensions.session_ticket', + }, + { + name: 'tls.server_hello.extensions.supported_versions', + type: 'alias', + path: 'tls.detailed.server_hello.extensions.supported_versions', + }, + { + name: 'tls.server_hello.extensions.ec_points_formats', + type: 'alias', + path: 'tls.detailed.server_hello.extensions.ec_points_formats', + }, + { + name: 'tls.server_hello.extensions._unparsed_', + type: 'alias', + path: 'tls.detailed.server_hello.extensions._unparsed_', + }, + { + name: 'tls.client_certificate.version', + type: 'alias', + path: 'tls.detailed.client_certificate.version', + }, + { + name: 'tls.client_certificate.serial_number', + type: 'alias', + path: 'tls.detailed.client_certificate.serial_number', + }, + { + name: 'tls.client_certificate.not_before', + type: 'alias', + path: 'tls.detailed.client_certificate.not_before', + }, + { + name: 'tls.client_certificate.not_after', + type: 'alias', + path: 'tls.detailed.client_certificate.not_after', + }, + { + name: 'tls.client_certificate.public_key_algorithm', + type: 'alias', + path: 'tls.detailed.client_certificate.public_key_algorithm', + }, + { + name: 'tls.client_certificate.public_key_size', + type: 'alias', + path: 'tls.detailed.client_certificate.public_key_size', + }, + { + name: 'tls.client_certificate.signature_algorithm', + type: 'alias', + path: 'tls.detailed.client_certificate.signature_algorithm', + }, + { + name: 'tls.client_certificate.alternative_names', + type: 'alias', + path: 'tls.detailed.client_certificate.alternative_names', + }, + { + name: 'tls.client_certificate.subject.country', + type: 'alias', + path: 'tls.detailed.client_certificate.subject.country', + }, + { + name: 'tls.client_certificate.subject.organization', + type: 'alias', + path: 'tls.detailed.client_certificate.subject.organization', + }, + { + name: 'tls.client_certificate.subject.organizational_unit', + type: 'alias', + path: 'tls.detailed.client_certificate.subject.organizational_unit', + }, + { + name: 'tls.client_certificate.subject.province', + type: 'alias', + path: 'tls.detailed.client_certificate.subject.province', + }, + { + name: 'tls.client_certificate.subject.common_name', + type: 'alias', + path: 'tls.detailed.client_certificate.subject.common_name', + }, + { + name: 'tls.client_certificate.subject.locality', + type: 'alias', + path: 'tls.detailed.client_certificate.subject.locality', + }, + { + name: 'tls.client_certificate.issuer.country', + type: 'alias', + path: 'tls.detailed.client_certificate.issuer.country', + }, + { + name: 'tls.client_certificate.issuer.organization', + type: 'alias', + path: 'tls.detailed.client_certificate.issuer.organization', + }, + { + name: 'tls.client_certificate.issuer.organizational_unit', + type: 'alias', + path: 'tls.detailed.client_certificate.issuer.organizational_unit', + }, + { + name: 'tls.client_certificate.issuer.province', + type: 'alias', + path: 'tls.detailed.client_certificate.issuer.province', + }, + { + name: 'tls.client_certificate.issuer.common_name', + type: 'alias', + path: 'tls.detailed.client_certificate.issuer.common_name', + }, + { + name: 'tls.client_certificate.issuer.locality', + type: 'alias', + path: 'tls.detailed.client_certificate.issuer.locality', + }, + { + name: 'tls.server_certificate.version', + type: 'alias', + path: 'tls.detailed.server_certificate.version', + }, + { + name: 'tls.server_certificate.serial_number', + type: 'alias', + path: 'tls.detailed.server_certificate.serial_number', + }, + { + name: 'tls.server_certificate.not_before', + type: 'alias', + path: 'tls.detailed.server_certificate.not_before', + }, + { + name: 'tls.server_certificate.not_after', + type: 'alias', + path: 'tls.detailed.server_certificate.not_after', + }, + { + name: 'tls.server_certificate.public_key_algorithm', + type: 'alias', + path: 'tls.detailed.server_certificate.public_key_algorithm', + }, + { + name: 'tls.server_certificate.public_key_size', + type: 'alias', + path: 'tls.detailed.server_certificate.public_key_size', + }, + { + name: 'tls.server_certificate.signature_algorithm', + type: 'alias', + path: 'tls.detailed.server_certificate.signature_algorithm', + }, + { + name: 'tls.server_certificate.alternative_names', + type: 'alias', + path: 'tls.detailed.server_certificate.alternative_names', + }, + { + name: 'tls.server_certificate.subject.country', + type: 'alias', + path: 'tls.detailed.server_certificate.subject.country', + }, + { + name: 'tls.server_certificate.subject.organization', + type: 'alias', + path: 'tls.detailed.server_certificate.subject.organization', + }, + { + name: 'tls.server_certificate.subject.organizational_unit', + type: 'alias', + path: 'tls.detailed.server_certificate.subject.organizational_unit', + }, + { + name: 'tls.server_certificate.subject.province', + type: 'alias', + path: 'tls.detailed.server_certificate.subject.province', + }, + { + name: 'tls.server_certificate.subject.common_name', + type: 'alias', + path: 'tls.detailed.server_certificate.subject.common_name', + }, + { + name: 'tls.server_certificate.subject.locality', + type: 'alias', + path: 'tls.detailed.server_certificate.subject.locality', + }, + { + name: 'tls.server_certificate.issuer.country', + type: 'alias', + path: 'tls.detailed.server_certificate.issuer.country', + }, + { + name: 'tls.server_certificate.issuer.organization', + type: 'alias', + path: 'tls.detailed.server_certificate.issuer.organization', + }, + { + name: 'tls.server_certificate.issuer.organizational_unit', + type: 'alias', + path: 'tls.detailed.server_certificate.issuer.organizational_unit', + }, + { + name: 'tls.server_certificate.issuer.province', + type: 'alias', + path: 'tls.detailed.server_certificate.issuer.province', + }, + { + name: 'tls.server_certificate.issuer.common_name', + type: 'alias', + path: 'tls.detailed.server_certificate.issuer.common_name', + }, + { + name: 'tls.server_certificate.issuer.locality', + type: 'alias', + path: 'tls.detailed.server_certificate.issuer.locality', + }, + { + name: 'tls.alert_types', + type: 'alias', + path: 'tls.detailed.alert_types', + }, ], }, ]; diff --git a/x-pack/legacy/plugins/siem/server/utils/beat_schema/index.test.ts b/x-pack/legacy/plugins/siem/server/utils/beat_schema/index.test.ts index 53ed7c26b877f..56ceca2b70e9c 100644 --- a/x-pack/legacy/plugins/siem/server/utils/beat_schema/index.test.ts +++ b/x-pack/legacy/plugins/siem/server/utils/beat_schema/index.test.ts @@ -17,175 +17,94 @@ describe('Schema Beat', () => { convertData[0].fields = isArray(convertData[0].fields) ? convertData[0].fields!.slice(0, 6) : []; + expect(convertSchemaToAssociativeArray(convertData)).toEqual({ '@timestamp': { description: - 'Date/time when the event originated. For log events this is the date/time when the event was generated, and not when it was read. Required field for all events.', + 'Date/time when the event originated.\n\nThis is the date/time extracted from the event, typically representing when\nthe event was generated by the source.\n\nIf the event source has no original timestamp, this value is typically populated\nby the first time the event was received by the pipeline.\n\nRequired field for all events.', example: '2016-05-23T08:05:34.853Z', name: '@timestamp', type: 'date', }, - tags: { - description: 'List of keywords used to tag each event.', - example: '["production", "env2"]', - name: 'tags', - type: 'keyword', - }, labels: { description: - 'Key/value pairs. Can be used to add meta information to events. Should not contain nested objects. All values are stored as keyword. Example: `docker` and `k8s` labels.', - example: '{"env":"production","application":"foo-bar"}', + 'Custom key/value pairs.\n\nCan be used to add meta information to events. Should not contain nested objects.\nAll values are stored as keyword.\n\nExample: `docker` and `k8s` labels.', + example: '{"application": "foo-bar", "env": "production"}', name: 'labels', type: 'object', }, message: { description: - 'For log events the message field contains the log message. In other use cases the message field can be used to concatenate different values which are then freely searchable. If multiple messages exist, they can be combined into one message.', + 'For log events the message field contains the log message, optimized\nfor viewing in a log viewer.\n\nFor structured logs without an original message field, other fields can be concatenated\nto form a human-readable summary of the event.\n\nIf multiple messages exist, they can be combined into one message.', example: 'Hello World', name: 'message', type: 'text', }, + tags: { + description: 'List of keywords used to tag each event.', + example: '["production", "env2"]', + name: 'tags', + type: 'keyword', + }, agent: { description: - 'The agent fields contain the data about the software entity, if any, that collects, detects, or observes events on a host, or takes measurements on a host. Examples include Beats. Agents may also run on observers. ECS agent.* fields shall be populated with details of the agent running on the host or observer where the event happened or the measurement was taken.', + 'The agent fields contain the data about the software entity, if\nany, that collects, detects, or observes events on a host, or takes measurements\non a host.\n\nExamples include Beats. Agents may also run on observers. ECS agent.* fields\nshall be populated with details of the agent running on the host or observer\nwhere the event happened or the measurement was taken.', name: 'agent', type: 'group', fields: { - 'agent.version': { - description: 'Version of the agent.', - example: '6.0.0-rc2', - name: 'version', + 'agent.ephemeral_id': { + description: + 'Ephemeral identifier of this agent (if one exists).\n\nThis id normally changes across restarts, but `agent.id` does not.', + example: '8a4f500f', + name: 'ephemeral_id', + type: 'keyword', + }, + 'agent.id': { + description: + 'Unique identifier of this agent (if one exists).\n\nExample: For Beats this would be beat.id.', + example: '8a4f500d', + name: 'id', type: 'keyword', }, 'agent.name': { description: - 'Name of the agent. This is a name that can be given to an agent. This can be helpful if for example two Filebeat instances are running on the same host but a human readable separation is needed on which Filebeat instance data is coming from. If no name is given, the name is often left empty.', + 'Custom name of the agent.\n\nThis is a name that can be given to an agent. This can be helpful if for example\ntwo Filebeat instances are running on the same host but a human readable separation\nis needed on which Filebeat instance data is coming from.\n\nIf no name is given, the name is often left empty.', example: 'foo', name: 'name', type: 'keyword', }, 'agent.type': { description: - 'Type of the agent. The agent type stays always the same and should be given by the agent used. In case of Filebeat the agent would always be Filebeat also if two Filebeat instances are run on the same machine.', + 'Type of the agent.\n\nThe agent type stays always the same and should be given by the agent used.\nIn case of Filebeat the agent would always be Filebeat also if two Filebeat\ninstances are run on the same machine.', example: 'filebeat', name: 'type', type: 'keyword', }, - 'agent.id': { - description: - 'Unique identifier of this agent (if one exists). Example: For Beats this would be beat.id.', - example: '8a4f500d', - name: 'id', - type: 'keyword', - }, - 'agent.ephemeral_id': { - description: - 'Ephemeral identifier of this agent (if one exists). This id normally changes across restarts, but `agent.id` does not.', - example: '8a4f500f', - name: 'ephemeral_id', + 'agent.version': { + description: 'Version of the agent.', + example: '6.0.0-rc2', + name: 'version', type: 'keyword', }, }, }, - client: { + as: { description: - 'A client is defined as the initiator of a network connection for events regarding sessions, connections, or bidirectional flow records. For TCP events, the client is the initiator of the TCP connection that sends the SYN packet(s). For other protocols, the client is generally the initiator or requestor in the network transaction. Some systems use the term "originator" to refer the client in TCP connections. The client fields describe details about the system acting as the client in the network event. Client fields are usually populated in conjunction with server fields. Client fields are generally not populated for packet-level events. Client / server representations can add semantic context to an exchange, which is helpful to visualize the data in certain situations. If your context falls in that category, you should still ensure that source and destination are filled appropriately.', - name: 'client', + 'An autonomous system (AS) is a collection of connected Internet Protocol\n(IP) routing prefixes under the control of one or more network operators on\nbehalf of a single administrative entity or domain that presents a common, clearly\ndefined routing policy to the internet.', + name: 'as', type: 'group', fields: { - 'client.address': { + 'as.number': { description: - 'Some event client addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is.', - name: 'address', - type: 'keyword', - }, - 'client.ip': { - description: - 'IP address of the client. Can be one or multiple IPv4 or IPv6 addresses.', - name: 'ip', - type: 'ip', - }, - 'client.port': { - description: 'Port of the client.', - name: 'port', + 'Unique number allocated to the autonomous system. The autonomous\nsystem number (ASN) uniquely identifies each network on the Internet.', + example: 15169, + name: 'number', type: 'long', }, - 'client.mac': { - description: 'MAC address of the client.', - name: 'mac', - type: 'keyword', - }, - 'client.domain': { - description: 'Client domain.', - name: 'domain', - type: 'keyword', - }, - 'client.bytes': { - description: 'Bytes sent from the client to the server.', - example: 184, - format: 'bytes', - name: 'bytes', - type: 'long', - }, - 'client.packets': { - description: 'Packets sent from the client to the server.', - example: 12, - name: 'packets', - type: 'long', - }, - 'client.geo': { - description: - 'Geo fields can carry data about a specific location related to an event or geo information derived from an IP field.', - name: 'geo', - type: 'group', - }, - 'client.geo.location': { - description: 'Longitude and latitude.', - example: '{ "lon": -73.614830, "lat": 45.505918 }', - name: 'location', - type: 'geo_point', - }, - 'client.geo.continent_name': { - description: 'Name of the continent.', - example: 'North America', - name: 'continent_name', - type: 'keyword', - }, - 'client.geo.country_name': { - description: 'Country name.', - example: 'Canada', - name: 'country_name', - type: 'keyword', - }, - 'client.geo.region_name': { - description: 'Region name.', - example: 'Quebec', - name: 'region_name', - type: 'keyword', - }, - 'client.geo.city_name': { - description: 'City name.', - example: 'Montreal', - name: 'city_name', - type: 'keyword', - }, - 'client.geo.country_iso_code': { - description: 'Country ISO code.', - example: 'CA', - name: 'country_iso_code', - type: 'keyword', - }, - 'client.geo.region_iso_code': { - description: 'Region ISO code.', - example: 'CA-QC', - name: 'region_iso_code', - type: 'keyword', - }, - 'client.geo.name': { - description: - 'User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation.', - example: 'boston-dc', - name: 'name', + 'as.organization.name': { + description: 'Organization name.', + example: 'Google LLC', + name: 'organization.name', type: 'keyword', }, }, @@ -198,175 +117,94 @@ describe('Schema Beat', () => { convertData[0].fields = isArray(convertData[0].fields) ? convertData[0].fields!.slice(0, 6) : []; + expect(convertSchemaToAssociativeArray(convertData)).toEqual({ '@timestamp': { description: - 'Date/time when the event originated. For log events this is the date/time when the event was generated, and not when it was read. Required field for all events.', + 'Date/time when the event originated.\n\nThis is the date/time extracted from the event, typically representing when\nthe event was generated by the source.\n\nIf the event source has no original timestamp, this value is typically populated\nby the first time the event was received by the pipeline.\n\nRequired field for all events.', example: '2016-05-23T08:05:34.853Z', name: '@timestamp', type: 'date', }, - tags: { - description: 'List of keywords used to tag each event.', - example: '["production", "env2"]', - name: 'tags', - type: 'keyword', - }, labels: { description: - 'Key/value pairs. Can be used to add meta information to events. Should not contain nested objects. All values are stored as keyword. Example: `docker` and `k8s` labels.', - example: '{"env":"production","application":"foo-bar"}', + 'Custom key/value pairs.\n\nCan be used to add meta information to events. Should not contain nested objects.\nAll values are stored as keyword.\n\nExample: `docker` and `k8s` labels.', + example: '{"application": "foo-bar", "env": "production"}', name: 'labels', type: 'object', }, message: { description: - 'For log events the message field contains the log message. In other use cases the message field can be used to concatenate different values which are then freely searchable. If multiple messages exist, they can be combined into one message.', + 'For log events the message field contains the log message, optimized\nfor viewing in a log viewer.\n\nFor structured logs without an original message field, other fields can be concatenated\nto form a human-readable summary of the event.\n\nIf multiple messages exist, they can be combined into one message.', example: 'Hello World', name: 'message', type: 'text', }, + tags: { + description: 'List of keywords used to tag each event.', + example: '["production", "env2"]', + name: 'tags', + type: 'keyword', + }, agent: { description: - 'The agent fields contain the data about the software entity, if any, that collects, detects, or observes events on a host, or takes measurements on a host. Examples include Beats. Agents may also run on observers. ECS agent.* fields shall be populated with details of the agent running on the host or observer where the event happened or the measurement was taken.', + 'The agent fields contain the data about the software entity, if\nany, that collects, detects, or observes events on a host, or takes measurements\non a host.\n\nExamples include Beats. Agents may also run on observers. ECS agent.* fields\nshall be populated with details of the agent running on the host or observer\nwhere the event happened or the measurement was taken.', name: 'agent', type: 'group', fields: { - 'agent.version': { - description: 'Version of the agent.', - example: '6.0.0-rc2', - name: 'version', + 'agent.ephemeral_id': { + description: + 'Ephemeral identifier of this agent (if one exists).\n\nThis id normally changes across restarts, but `agent.id` does not.', + example: '8a4f500f', + name: 'ephemeral_id', + type: 'keyword', + }, + 'agent.id': { + description: + 'Unique identifier of this agent (if one exists).\n\nExample: For Beats this would be beat.id.', + example: '8a4f500d', + name: 'id', type: 'keyword', }, 'agent.name': { description: - 'Name of the agent. This is a name that can be given to an agent. This can be helpful if for example two Filebeat instances are running on the same host but a human readable separation is needed on which Filebeat instance data is coming from. If no name is given, the name is often left empty.', + 'Custom name of the agent.\n\nThis is a name that can be given to an agent. This can be helpful if for example\ntwo Filebeat instances are running on the same host but a human readable separation\nis needed on which Filebeat instance data is coming from.\n\nIf no name is given, the name is often left empty.', example: 'foo', name: 'name', type: 'keyword', }, 'agent.type': { description: - 'Type of the agent. The agent type stays always the same and should be given by the agent used. In case of Filebeat the agent would always be Filebeat also if two Filebeat instances are run on the same machine.', + 'Type of the agent.\n\nThe agent type stays always the same and should be given by the agent used.\nIn case of Filebeat the agent would always be Filebeat also if two Filebeat\ninstances are run on the same machine.', example: 'filebeat', name: 'type', type: 'keyword', }, - 'agent.id': { - description: - 'Unique identifier of this agent (if one exists). Example: For Beats this would be beat.id.', - example: '8a4f500d', - name: 'id', - type: 'keyword', - }, - 'agent.ephemeral_id': { - description: - 'Ephemeral identifier of this agent (if one exists). This id normally changes across restarts, but `agent.id` does not.', - example: '8a4f500f', - name: 'ephemeral_id', + 'agent.version': { + description: 'Version of the agent.', + example: '6.0.0-rc2', + name: 'version', type: 'keyword', }, }, }, - client: { + as: { description: - 'A client is defined as the initiator of a network connection for events regarding sessions, connections, or bidirectional flow records. For TCP events, the client is the initiator of the TCP connection that sends the SYN packet(s). For other protocols, the client is generally the initiator or requestor in the network transaction. Some systems use the term "originator" to refer the client in TCP connections. The client fields describe details about the system acting as the client in the network event. Client fields are usually populated in conjunction with server fields. Client fields are generally not populated for packet-level events. Client / server representations can add semantic context to an exchange, which is helpful to visualize the data in certain situations. If your context falls in that category, you should still ensure that source and destination are filled appropriately.', - name: 'client', + 'An autonomous system (AS) is a collection of connected Internet Protocol\n(IP) routing prefixes under the control of one or more network operators on\nbehalf of a single administrative entity or domain that presents a common, clearly\ndefined routing policy to the internet.', + name: 'as', type: 'group', fields: { - 'client.address': { + 'as.number': { description: - 'Some event client addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is.', - name: 'address', - type: 'keyword', - }, - 'client.ip': { - description: - 'IP address of the client. Can be one or multiple IPv4 or IPv6 addresses.', - name: 'ip', - type: 'ip', - }, - 'client.port': { - description: 'Port of the client.', - name: 'port', - type: 'long', - }, - 'client.mac': { - description: 'MAC address of the client.', - name: 'mac', - type: 'keyword', - }, - 'client.domain': { - description: 'Client domain.', - name: 'domain', - type: 'keyword', - }, - 'client.bytes': { - description: 'Bytes sent from the client to the server.', - example: 184, - format: 'bytes', - name: 'bytes', - type: 'long', - }, - 'client.packets': { - description: 'Packets sent from the client to the server.', - example: 12, - name: 'packets', + 'Unique number allocated to the autonomous system. The autonomous\nsystem number (ASN) uniquely identifies each network on the Internet.', + example: 15169, + name: 'number', type: 'long', }, - 'client.geo': { - description: - 'Geo fields can carry data about a specific location related to an event or geo information derived from an IP field.', - name: 'geo', - type: 'group', - }, - 'client.geo.location': { - description: 'Longitude and latitude.', - example: '{ "lon": -73.614830, "lat": 45.505918 }', - name: 'location', - type: 'geo_point', - }, - 'client.geo.continent_name': { - description: 'Name of the continent.', - example: 'North America', - name: 'continent_name', - type: 'keyword', - }, - 'client.geo.country_name': { - description: 'Country name.', - example: 'Canada', - name: 'country_name', - type: 'keyword', - }, - 'client.geo.region_name': { - description: 'Region name.', - example: 'Quebec', - name: 'region_name', - type: 'keyword', - }, - 'client.geo.city_name': { - description: 'City name.', - example: 'Montreal', - name: 'city_name', - type: 'keyword', - }, - 'client.geo.country_iso_code': { - description: 'Country ISO code.', - example: 'CA', - name: 'country_iso_code', - type: 'keyword', - }, - 'client.geo.region_iso_code': { - description: 'Region ISO code.', - example: 'CA-QC', - name: 'region_iso_code', - type: 'keyword', - }, - 'client.geo.name': { - description: - 'User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation.', - example: 'boston-dc', - name: 'name', + 'as.organization.name': { + description: 'Organization name.', + example: 'Google LLC', + name: 'organization.name', type: 'keyword', }, }, @@ -379,175 +217,94 @@ describe('Schema Beat', () => { convertData[0].fields = isArray(convertData[0].fields) ? convertData[0].fields!.slice(0, 6) : []; + expect(convertSchemaToAssociativeArray(convertData)).toEqual({ '@timestamp': { description: - 'Date/time when the event originated. For log events this is the date/time when the event was generated, and not when it was read. Required field for all events.', + 'Date/time when the event originated.\n\nThis is the date/time extracted from the event, typically representing when\nthe event was generated by the source.\n\nIf the event source has no original timestamp, this value is typically populated\nby the first time the event was received by the pipeline.\n\nRequired field for all events.', example: '2016-05-23T08:05:34.853Z', name: '@timestamp', type: 'date', }, - tags: { - description: 'List of keywords used to tag each event.', - example: '["production", "env2"]', - name: 'tags', - type: 'keyword', - }, labels: { description: - 'Key/value pairs. Can be used to add meta information to events. Should not contain nested objects. All values are stored as keyword. Example: `docker` and `k8s` labels.', - example: '{"env":"production","application":"foo-bar"}', + 'Custom key/value pairs.\n\nCan be used to add meta information to events. Should not contain nested objects.\nAll values are stored as keyword.\n\nExample: `docker` and `k8s` labels.', + example: '{"application": "foo-bar", "env": "production"}', name: 'labels', type: 'object', }, message: { description: - 'For log events the message field contains the log message. In other use cases the message field can be used to concatenate different values which are then freely searchable. If multiple messages exist, they can be combined into one message.', + 'For log events the message field contains the log message, optimized\nfor viewing in a log viewer.\n\nFor structured logs without an original message field, other fields can be concatenated\nto form a human-readable summary of the event.\n\nIf multiple messages exist, they can be combined into one message.', example: 'Hello World', name: 'message', type: 'text', }, + tags: { + description: 'List of keywords used to tag each event.', + example: '["production", "env2"]', + name: 'tags', + type: 'keyword', + }, agent: { description: - 'The agent fields contain the data about the software entity, if any, that collects, detects, or observes events on a host, or takes measurements on a host. Examples include Beats. Agents may also run on observers. ECS agent.* fields shall be populated with details of the agent running on the host or observer where the event happened or the measurement was taken.', + 'The agent fields contain the data about the software entity, if\nany, that collects, detects, or observes events on a host, or takes measurements\non a host.\n\nExamples include Beats. Agents may also run on observers. ECS agent.* fields\nshall be populated with details of the agent running on the host or observer\nwhere the event happened or the measurement was taken.', name: 'agent', type: 'group', fields: { - 'agent.version': { - description: 'Version of the agent.', - example: '6.0.0-rc2', - name: 'version', + 'agent.ephemeral_id': { + description: + 'Ephemeral identifier of this agent (if one exists).\n\nThis id normally changes across restarts, but `agent.id` does not.', + example: '8a4f500f', + name: 'ephemeral_id', + type: 'keyword', + }, + 'agent.id': { + description: + 'Unique identifier of this agent (if one exists).\n\nExample: For Beats this would be beat.id.', + example: '8a4f500d', + name: 'id', type: 'keyword', }, 'agent.name': { description: - 'Name of the agent. This is a name that can be given to an agent. This can be helpful if for example two Filebeat instances are running on the same host but a human readable separation is needed on which Filebeat instance data is coming from. If no name is given, the name is often left empty.', + 'Custom name of the agent.\n\nThis is a name that can be given to an agent. This can be helpful if for example\ntwo Filebeat instances are running on the same host but a human readable separation\nis needed on which Filebeat instance data is coming from.\n\nIf no name is given, the name is often left empty.', example: 'foo', name: 'name', type: 'keyword', }, 'agent.type': { description: - 'Type of the agent. The agent type stays always the same and should be given by the agent used. In case of Filebeat the agent would always be Filebeat also if two Filebeat instances are run on the same machine.', + 'Type of the agent.\n\nThe agent type stays always the same and should be given by the agent used.\nIn case of Filebeat the agent would always be Filebeat also if two Filebeat\ninstances are run on the same machine.', example: 'filebeat', name: 'type', type: 'keyword', }, - 'agent.id': { - description: - 'Unique identifier of this agent (if one exists). Example: For Beats this would be beat.id.', - example: '8a4f500d', - name: 'id', - type: 'keyword', - }, - 'agent.ephemeral_id': { - description: - 'Ephemeral identifier of this agent (if one exists). This id normally changes across restarts, but `agent.id` does not.', - example: '8a4f500f', - name: 'ephemeral_id', + 'agent.version': { + description: 'Version of the agent.', + example: '6.0.0-rc2', + name: 'version', type: 'keyword', }, }, }, - client: { + as: { description: - 'A client is defined as the initiator of a network connection for events regarding sessions, connections, or bidirectional flow records. For TCP events, the client is the initiator of the TCP connection that sends the SYN packet(s). For other protocols, the client is generally the initiator or requestor in the network transaction. Some systems use the term "originator" to refer the client in TCP connections. The client fields describe details about the system acting as the client in the network event. Client fields are usually populated in conjunction with server fields. Client fields are generally not populated for packet-level events. Client / server representations can add semantic context to an exchange, which is helpful to visualize the data in certain situations. If your context falls in that category, you should still ensure that source and destination are filled appropriately.', - name: 'client', + 'An autonomous system (AS) is a collection of connected Internet Protocol\n(IP) routing prefixes under the control of one or more network operators on\nbehalf of a single administrative entity or domain that presents a common, clearly\ndefined routing policy to the internet.', + name: 'as', type: 'group', fields: { - 'client.address': { + 'as.number': { description: - 'Some event client addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is.', - name: 'address', - type: 'keyword', - }, - 'client.ip': { - description: - 'IP address of the client. Can be one or multiple IPv4 or IPv6 addresses.', - name: 'ip', - type: 'ip', - }, - 'client.port': { - description: 'Port of the client.', - name: 'port', - type: 'long', - }, - 'client.mac': { - description: 'MAC address of the client.', - name: 'mac', - type: 'keyword', - }, - 'client.domain': { - description: 'Client domain.', - name: 'domain', - type: 'keyword', - }, - 'client.bytes': { - description: 'Bytes sent from the client to the server.', - example: 184, - format: 'bytes', - name: 'bytes', + 'Unique number allocated to the autonomous system. The autonomous\nsystem number (ASN) uniquely identifies each network on the Internet.', + example: 15169, + name: 'number', type: 'long', }, - 'client.packets': { - description: 'Packets sent from the client to the server.', - example: 12, - name: 'packets', - type: 'long', - }, - 'client.geo': { - description: - 'Geo fields can carry data about a specific location related to an event or geo information derived from an IP field.', - name: 'geo', - type: 'group', - }, - 'client.geo.location': { - description: 'Longitude and latitude.', - example: '{ "lon": -73.614830, "lat": 45.505918 }', - name: 'location', - type: 'geo_point', - }, - 'client.geo.continent_name': { - description: 'Name of the continent.', - example: 'North America', - name: 'continent_name', - type: 'keyword', - }, - 'client.geo.country_name': { - description: 'Country name.', - example: 'Canada', - name: 'country_name', - type: 'keyword', - }, - 'client.geo.region_name': { - description: 'Region name.', - example: 'Quebec', - name: 'region_name', - type: 'keyword', - }, - 'client.geo.city_name': { - description: 'City name.', - example: 'Montreal', - name: 'city_name', - type: 'keyword', - }, - 'client.geo.country_iso_code': { - description: 'Country ISO code.', - example: 'CA', - name: 'country_iso_code', - type: 'keyword', - }, - 'client.geo.region_iso_code': { - description: 'Region ISO code.', - example: 'CA-QC', - name: 'region_iso_code', - type: 'keyword', - }, - 'client.geo.name': { - description: - 'User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation.', - example: 'boston-dc', - name: 'name', + 'as.organization.name': { + description: 'Organization name.', + example: 'Google LLC', + name: 'organization.name', type: 'keyword', }, }, @@ -562,40 +319,58 @@ describe('Schema Beat', () => { '_id', '_index', '@timestamp', - 'tags', 'labels', 'message', + 'tags', 'agent', + 'as', 'client', 'cloud', + 'code_signature', 'container', 'destination', + 'dll', + 'dns', 'ecs', 'error', 'event', 'file', + 'geo', 'group', + 'hash', 'host', 'http', + 'interface', 'log', 'network', 'observer', 'organization', 'os', + 'package', + 'pe', 'process', + 'registry', 'related', + 'rule', 'server', 'service', 'source', + 'threat', + 'tls', + 'tracing', 'url', 'user', 'user_agent', + 'vlan', + 'vulnerability', 'agent.hostname', 'beat.timezone', 'fields', 'beat.name', 'beat.hostname', + 'timeseries.instance', 'cloud.project.id', + 'cloud.image.id', 'meta.cloud.provider', 'meta.cloud.instance_id', 'meta.cloud.instance_name', @@ -605,55 +380,17 @@ describe('Schema Beat', () => { 'meta.cloud.region', 'docker', 'kubernetes', - 'type', - 'server.process.name', - 'server.process.args', - 'server.process.executable', - 'server.process.working_directory', - 'server.process.start', - 'client.process.name', - 'client.process.args', - 'client.process.executable', - 'client.process.working_directory', - 'client.process.start', - 'real_ip', - 'transport', - 'flow.final', - 'flow.id', - 'flow.vlan', - 'flow_id', - 'final', - 'vlan', - 'source.stats.net_bytes_total', - 'source.stats.net_packets_total', - 'dest.stats.net_bytes_total', - 'dest.stats.net_packets_total', - 'status', - 'method', - 'resource', - 'path', - 'query', - 'params', - 'notes', - 'request', - 'response', - 'bytes_in', - 'bytes_out', - 'amqp', - 'no_request', - 'cassandra', - 'dhcpv4', - 'dns', - 'icmp', - 'memcache', - 'mongodb', - 'mysql', - 'nfs', - 'rpc', - 'pgsql', - 'redis', - 'thrift', - 'tls', + 'jolokia.agent.version', + 'jolokia.agent.id', + 'jolokia.server.product', + 'jolokia.server.version', + 'jolokia.server.vendor', + 'jolokia.url', + 'jolokia.secured', + 'auditd', + 'geoip', + 'socket', + 'system.audit', ]); }); }); diff --git a/x-pack/legacy/plugins/siem/server/utils/beat_schema/index.ts b/x-pack/legacy/plugins/siem/server/utils/beat_schema/index.ts index a191bd835a7c7..3e9be4e265e63 100644 --- a/x-pack/legacy/plugins/siem/server/utils/beat_schema/index.ts +++ b/x-pack/legacy/plugins/siem/server/utils/beat_schema/index.ts @@ -106,19 +106,14 @@ export const getIndexSchemaDoc = memoize((index: string) => { ...extraSchemaField, ...convertSchemaToAssociativeArray(winlogbeatSchema), }; - } else if (index.match('ecs') != null) { - return { - ...extraSchemaField, - ...ecsSchema, - }; } - return {}; + return { + ...extraSchemaField, + ...convertSchemaToAssociativeArray(ecsSchema), + }; }); export const hasDocumentation = (index: string, path: string): boolean => { - if (index === 'unknown') { - return false; - } const splitPath = path.split('.'); const category = splitPath.length > 0 ? splitPath[0] : null; if (category === null) { @@ -131,16 +126,13 @@ export const hasDocumentation = (index: string, path: string): boolean => { }; export const getDocumentation = (index: string, path: string) => { - if (index === 'unknown') { - return ''; - } const splitPath = path.split('.'); const category = splitPath.length > 0 ? splitPath[0] : null; if (category === null) { - return ''; + return {}; } if (splitPath.length > 1) { - return get([category, 'fields', path], getIndexSchemaDoc(index)) || ''; + return get([category, 'fields', path], getIndexSchemaDoc(index)) || {}; } - return get(category, getIndexSchemaDoc(index)) || ''; + return get(category, getIndexSchemaDoc(index)) || {}; }; diff --git a/x-pack/legacy/plugins/siem/server/utils/beat_schema/type.ts b/x-pack/legacy/plugins/siem/server/utils/beat_schema/type.ts index f34519da34ee8..2b7be8f4b7539 100644 --- a/x-pack/legacy/plugins/siem/server/utils/beat_schema/type.ts +++ b/x-pack/legacy/plugins/siem/server/utils/beat_schema/type.ts @@ -12,25 +12,35 @@ export type IndexAlias = 'auditbeat' | 'filebeat' | 'packetbeat' | 'ecs' | 'winl */ export interface SchemaFields { + default_field: boolean; + default_fields: boolean; definition: string; + deprecated: string; description: string; doc_values: boolean; - example: string | number | object; + example: string | number | object | boolean; footnote: string; format: string; group: number; index: boolean; + ignore_above: number; input_format: string; level: string; migration: boolean; multi_fields: object[]; name: string; + norms: boolean; object_type: string; + object_type_mapping_type: string; + output_format: string; + output_precision: number; + overwrite: boolean; path: string; possible_values: string[] | number[]; release: string; required: boolean; reusable: object; + short: string; title: string; type: string; fields: Array>; @@ -48,33 +58,6 @@ export interface SchemaItem { export type Schema = Array>; -/* - * ECS Interface - * - */ - -interface EcsField { - description: string; - example: string; - footnote: string; - group: number; - level: string; - name: string; - required: boolean; - type: string; -} - -interface EcsNamespace { - description: string; - fields: Readonly>; - group: number; - name: string; - title: string; - type: string; -} - -export type EcsSchema = Readonly>; - /* * Associative Array Output Interface * diff --git a/x-pack/legacy/plugins/uptime/common/constants/rest_api.ts b/x-pack/legacy/plugins/uptime/common/constants/rest_api.ts index 86e2b03e13f22..dffa131870db1 100644 --- a/x-pack/legacy/plugins/uptime/common/constants/rest_api.ts +++ b/x-pack/legacy/plugins/uptime/common/constants/rest_api.ts @@ -15,7 +15,7 @@ export enum API_URLS { PING_HISTOGRAM = `/api/uptime/ping/histogram`, SNAPSHOT_COUNT = `/api/uptime/snapshot/count`, FILTERS = `/api/uptime/filters`, - logPageView = `/api/uptime/logPageView`, + LOG_PAGE_VIEW = `/api/uptime/log_page_view`, ML_MODULE_JOBS = `/api/ml/modules/jobs_exist/`, ML_SETUP_MODULE = '/api/ml/modules/setup/', diff --git a/x-pack/legacy/plugins/uptime/public/hooks/use_telemetry.ts b/x-pack/legacy/plugins/uptime/public/hooks/use_telemetry.ts index fc0e0ce1c3e88..13fe523332ae5 100644 --- a/x-pack/legacy/plugins/uptime/public/hooks/use_telemetry.ts +++ b/x-pack/legacy/plugins/uptime/public/hooks/use_telemetry.ts @@ -30,6 +30,6 @@ export const useUptimeTelemetry = (page?: UptimePage) => { dateEnd: dateRangeEnd, autoRefreshEnabled: !autorefreshIsPaused, }; - apiService.post(API_URLS.logPageView, params); + apiService.post(API_URLS.LOG_PAGE_VIEW, params); }, [autorefreshInterval, autorefreshIsPaused, dateRangeEnd, dateRangeStart, page]); }; diff --git a/x-pack/plugins/actions/server/builtin_action_types/lib/send_email.ts b/x-pack/plugins/actions/server/builtin_action_types/lib/send_email.ts index 74eead0708545..47d7aff8022ce 100644 --- a/x-pack/plugins/actions/server/builtin_action_types/lib/send_email.ts +++ b/x-pack/plugins/actions/server/builtin_action_types/lib/send_email.ts @@ -65,6 +65,11 @@ export async function sendEmail(logger: Logger, options: SendEmailOptions): Prom transportConfig.host = host; transportConfig.port = port; transportConfig.secure = !!secure; + if (!transportConfig.secure) { + transportConfig.tls = { + rejectUnauthorized: false, + }; + } } const nodemailerTransport = nodemailer.createTransport(transportConfig); diff --git a/x-pack/plugins/apm/common/agent_configuration/amount_and_unit.ts b/x-pack/plugins/apm/common/agent_configuration/amount_and_unit.ts index 447e529c9c199..d6520ae150539 100644 --- a/x-pack/plugins/apm/common/agent_configuration/amount_and_unit.ts +++ b/x-pack/plugins/apm/common/agent_configuration/amount_and_unit.ts @@ -10,7 +10,8 @@ interface AmountAndUnit { } export function amountAndUnitToObject(value: string): AmountAndUnit { - const [, amount = '', unit = ''] = value.match(/(\d+)?(\w+)?/) || []; + // matches any postive and negative number and its unit. + const [, amount = '', unit = ''] = value.match(/(^-?\d+)?(\w+)?/) || []; return { amount, unit }; } diff --git a/x-pack/plugins/apm/common/agent_configuration/runtime_types/duration_rt.test.ts b/x-pack/plugins/apm/common/agent_configuration/runtime_types/duration_rt.test.ts index a83ee9262cad6..98d0cb5f028c3 100644 --- a/x-pack/plugins/apm/common/agent_configuration/runtime_types/duration_rt.test.ts +++ b/x-pack/plugins/apm/common/agent_configuration/runtime_types/duration_rt.test.ts @@ -10,24 +10,56 @@ * you may not use this file except in compliance with the Elastic License. */ -import { durationRt } from './duration_rt'; +import { durationRt, getDurationRt } from './duration_rt'; import { isRight } from 'fp-ts/lib/Either'; describe('durationRt', () => { describe('it should not accept', () => { - [undefined, null, '', 0, 'foo', true, false, '100', 's', 'm', '0h'].map( + [ + undefined, + null, + '', + 0, + 'foo', + true, + false, + '100', + 's', + 'm', + '0ms', + '-1ms' + ].map(input => { + it(`${JSON.stringify(input)}`, () => { + expect(isRight(durationRt.decode(input))).toBe(false); + }); + }); + }); + + describe('it should accept', () => { + ['1000ms', '2s', '3m', '1s'].map(input => { + it(`${JSON.stringify(input)}`, () => { + expect(isRight(durationRt.decode(input))).toBe(true); + }); + }); + }); +}); + +describe('getDurationRt', () => { + const customDurationRt = getDurationRt({ min: -1 }); + describe('it should not accept', () => { + [undefined, null, '', 0, 'foo', true, false, '100', 's', 'm', '-2ms'].map( input => { it(`${JSON.stringify(input)}`, () => { - expect(isRight(durationRt.decode(input))).toBe(false); + expect(isRight(customDurationRt.decode(input))).toBe(false); }); } ); }); - describe('It should accept', () => { - ['1000ms', '2s', '3m'].map(input => { + describe('it should accept', () => { + ['1000ms', '2s', '3m', '1s', '-1s', '0ms'].map(input => { it(`${JSON.stringify(input)}`, () => { - expect(isRight(durationRt.decode(input))).toBe(true); + expect(isRight(customDurationRt.decode(input))).toBe(true); }); }); }); diff --git a/x-pack/plugins/apm/common/agent_configuration/runtime_types/duration_rt.ts b/x-pack/plugins/apm/common/agent_configuration/runtime_types/duration_rt.ts index 383fd69be9a78..b691276854fb0 100644 --- a/x-pack/plugins/apm/common/agent_configuration/runtime_types/duration_rt.ts +++ b/x-pack/plugins/apm/common/agent_configuration/runtime_types/duration_rt.ts @@ -10,24 +10,28 @@ import { amountAndUnitToObject } from '../amount_and_unit'; export const DURATION_UNITS = ['ms', 's', 'm']; -export const durationRt = new t.Type( - 'durationRt', - t.string.is, - (input, context) => { - return either.chain(t.string.validate(input, context), inputAsString => { - const { amount, unit } = amountAndUnitToObject(inputAsString); - const amountAsInt = parseInt(amount, 10); - const isValidUnit = DURATION_UNITS.includes(unit); - const isValid = amountAsInt > 0 && isValidUnit; +export function getDurationRt({ min }: { min: number }) { + return new t.Type( + 'durationRt', + t.string.is, + (input, context) => { + return either.chain(t.string.validate(input, context), inputAsString => { + const { amount, unit } = amountAndUnitToObject(inputAsString); + const amountAsInt = parseInt(amount, 10); + const isValidUnit = DURATION_UNITS.includes(unit); + const isValid = amountAsInt >= min && isValidUnit; - return isValid - ? t.success(inputAsString) - : t.failure( - input, - context, - `Must have numeric amount and a valid unit (${DURATION_UNITS})` - ); - }); - }, - t.identity -); + return isValid + ? t.success(inputAsString) + : t.failure( + input, + context, + `Must have numeric amount and a valid unit (${DURATION_UNITS})` + ); + }); + }, + t.identity + ); +} + +export const durationRt = getDurationRt({ min: 1 }); diff --git a/x-pack/plugins/apm/common/agent_configuration/setting_definitions/__snapshots__/index.test.ts.snap b/x-pack/plugins/apm/common/agent_configuration/setting_definitions/__snapshots__/index.test.ts.snap index 4b74b07fc8e27..bc435179762a2 100644 --- a/x-pack/plugins/apm/common/agent_configuration/setting_definitions/__snapshots__/index.test.ts.snap +++ b/x-pack/plugins/apm/common/agent_configuration/setting_definitions/__snapshots__/index.test.ts.snap @@ -118,6 +118,7 @@ Array [ }, Object { "key": "span_frames_min_duration", + "min": -1, "type": "duration", "units": Array [ "ms", diff --git a/x-pack/plugins/apm/common/agent_configuration/setting_definitions/general_settings.ts b/x-pack/plugins/apm/common/agent_configuration/setting_definitions/general_settings.ts index 152db37a1bff3..f1d11c5c70c2b 100644 --- a/x-pack/plugins/apm/common/agent_configuration/setting_definitions/general_settings.ts +++ b/x-pack/plugins/apm/common/agent_configuration/setting_definitions/general_settings.ts @@ -8,6 +8,7 @@ import { i18n } from '@kbn/i18n'; import { getIntegerRt } from '../runtime_types/integer_rt'; import { captureBodyRt } from '../runtime_types/capture_body_rt'; import { RawSettingDefinition } from './types'; +import { getDurationRt } from '../runtime_types/duration_rt'; /* * Settings added here will show up in the UI and will be validated on the client and server @@ -143,6 +144,7 @@ export const generalSettings: RawSettingDefinition[] = [ { key: 'span_frames_min_duration', type: 'duration', + validation: getDurationRt({ min: -1 }), defaultValue: '5ms', label: i18n.translate('xpack.apm.agentConfig.spanFramesMinDuration.label', { defaultMessage: 'Span frames minimum duration' @@ -154,7 +156,8 @@ export const generalSettings: RawSettingDefinition[] = [ 'In its default settings, the APM agent will collect a stack trace with every recorded span.\nWhile this is very helpful to find the exact place in your code that causes the span, collecting this stack trace does have some overhead. \nWhen setting this option to a negative value, like `-1ms`, stack traces will be collected for all spans. Setting it to a positive value, e.g. `5ms`, will limit stack trace collection to spans with durations equal to or longer than the given value, e.g. 5 milliseconds.\n\nTo disable stack trace collection for spans completely, set the value to `0ms`.' } ), - excludeAgents: ['js-base', 'rum-js', 'nodejs'] + excludeAgents: ['js-base', 'rum-js', 'nodejs'], + min: -1 }, // STACK_TRACE_LIMIT diff --git a/x-pack/plugins/apm/common/agent_configuration/setting_definitions/types.d.ts b/x-pack/plugins/apm/common/agent_configuration/setting_definitions/types.d.ts index 6b584fc7e2048..282ced346dda0 100644 --- a/x-pack/plugins/apm/common/agent_configuration/setting_definitions/types.d.ts +++ b/x-pack/plugins/apm/common/agent_configuration/setting_definitions/types.d.ts @@ -91,6 +91,7 @@ interface BytesSetting extends BaseSetting { interface DurationSetting extends BaseSetting { type: 'duration'; units?: string[]; + min?: number; } export type RawSettingDefinition = diff --git a/x-pack/plugins/apm/server/lib/service_map/get_service_map_service_node_info.ts b/x-pack/plugins/apm/server/lib/service_map/get_service_map_service_node_info.ts index 0fe825e8ace35..d7e28828572d5 100644 --- a/x-pack/plugins/apm/server/lib/service_map/get_service_map_service_node_info.ts +++ b/x-pack/plugins/apm/server/lib/service_map/get_service_map_service_node_info.ts @@ -41,9 +41,7 @@ export async function getServiceMapServiceNodeInfo({ const filter: ESFilter[] = [ { range: rangeFilter(start, end) }, { term: { [SERVICE_NAME]: serviceName } }, - ...(environment - ? [{ term: { [SERVICE_ENVIRONMENT]: SERVICE_ENVIRONMENT } }] - : []) + ...(environment ? [{ term: { [SERVICE_ENVIRONMENT]: environment } }] : []) ]; const minutes = Math.abs((end - start) / (1000 * 60)); diff --git a/x-pack/plugins/data_enhanced/server/search/es_search_strategy.test.ts b/x-pack/plugins/data_enhanced/server/search/es_search_strategy.test.ts index 88c576c70bdf0..a8a42fe11e7ce 100644 --- a/x-pack/plugins/data_enhanced/server/search/es_search_strategy.test.ts +++ b/x-pack/plugins/data_enhanced/server/search/es_search_strategy.test.ts @@ -71,7 +71,7 @@ describe('ES search strategy', () => { expect(mockApiCaller.mock.calls[0][0]).toBe('transport.request'); const { method, path, body } = mockApiCaller.mock.calls[0][1]; expect(method).toBe('POST'); - expect(path).toBe('logstash-*/_async_search'); + expect(path).toBe('/logstash-*/_async_search'); expect(body).toEqual({ query: {} }); }); @@ -94,7 +94,7 @@ describe('ES search strategy', () => { expect(mockApiCaller.mock.calls[0][0]).toBe('transport.request'); const { method, path, body } = mockApiCaller.mock.calls[0][1]; expect(method).toBe('GET'); - expect(path).toBe('_async_search/foo'); + expect(path).toBe('/_async_search/foo'); expect(body).toEqual(undefined); }); @@ -117,7 +117,7 @@ describe('ES search strategy', () => { expect(mockApiCaller.mock.calls[0][0]).toBe('transport.request'); const { method, path } = mockApiCaller.mock.calls[0][1]; expect(method).toBe('POST'); - expect(path).toBe('foo-%E7%A8%8B/_async_search'); + expect(path).toBe('/foo-%E7%A8%8B/_async_search'); }); it('calls the rollup API if the index is a rollup type', async () => { @@ -139,6 +139,6 @@ describe('ES search strategy', () => { expect(mockApiCaller.mock.calls[0][0]).toBe('transport.request'); const { method, path } = mockApiCaller.mock.calls[0][1]; expect(method).toBe('POST'); - expect(path).toBe('foo-%E7%A8%8B/_rollup_search'); + expect(path).toBe('/foo-%E7%A8%8B/_rollup_search'); }); }); diff --git a/x-pack/plugins/data_enhanced/server/search/es_search_strategy.ts b/x-pack/plugins/data_enhanced/server/search/es_search_strategy.ts index 301f184af7d81..6b329bccab4a7 100644 --- a/x-pack/plugins/data_enhanced/server/search/es_search_strategy.ts +++ b/x-pack/plugins/data_enhanced/server/search/es_search_strategy.ts @@ -45,7 +45,7 @@ export const enhancedEsSearchStrategyProvider: TSearchStrategyProvider = async id => { const method = 'DELETE'; - const path = encodeURI(`_async_search/${id}`); + const path = encodeURI(`/_async_search/${id}`); await caller('transport.request', { method, path }); }; @@ -66,7 +66,7 @@ async function asyncSearch( const { body = undefined, index = undefined, ...queryParams } = request.id ? {} : params; const method = request.id ? 'GET' : 'POST'; - const path = encodeURI(request.id ? `_async_search/${request.id}` : `${index}/_async_search`); + const path = encodeURI(request.id ? `/_async_search/${request.id}` : `/${index}/_async_search`); // Wait up to 1s for the response to return const query = toSnakeCase({ waitForCompletionTimeout: '1s', ...queryParams }); @@ -87,7 +87,7 @@ async function rollupSearch( ) { const { body, index, ...params } = request.params; const method = 'POST'; - const path = encodeURI(`${index}/_rollup_search`); + const path = encodeURI(`/${index}/_rollup_search`); const query = toSnakeCase(params); const rawResponse = await ((caller( diff --git a/x-pack/plugins/endpoint/common/generate_data.ts b/x-pack/plugins/endpoint/common/generate_data.ts index 0ec105129b7ac..7c24bd9d77148 100644 --- a/x-pack/plugins/endpoint/common/generate_data.ts +++ b/x-pack/plugins/endpoint/common/generate_data.ts @@ -7,6 +7,11 @@ import uuid from 'uuid'; import seedrandom from 'seedrandom'; import { AlertEvent, EndpointEvent, HostMetadata, OSFields, HostFields } from './types'; +// FIXME: move types/model to top-level +// eslint-disable-next-line @kbn/eslint/no-restricted-paths +import { PolicyData } from '../public/applications/endpoint/types'; +// eslint-disable-next-line @kbn/eslint/no-restricted-paths +import { generatePolicy } from '../public/applications/endpoint/models/policy'; export type Event = AlertEvent | EndpointEvent; @@ -452,6 +457,39 @@ export class EndpointDocGenerator { } } + /** + * Generates an Ingest `datasource` that includes the Endpoint Policy data + */ + public generatePolicyDatasource(): PolicyData { + return { + id: this.seededUUIDv4(), + name: 'Endpoint Policy', + description: 'Policy to protect the worlds data', + config_id: this.seededUUIDv4(), + enabled: true, + output_id: '', + inputs: [ + { + type: 'endpoint', + enabled: true, + streams: [], + config: { + policy: { + value: generatePolicy(), + }, + }, + }, + ], + namespace: 'default', + package: { + name: 'endpoint', + title: 'Elastic Endpoint', + version: '1.0.0', + }, + revision: 1, + }; + } + private randomN(n: number): number { return Math.floor(this.random() * n); } diff --git a/x-pack/plugins/endpoint/public/applications/endpoint/index.tsx b/x-pack/plugins/endpoint/public/applications/endpoint/index.tsx index fa9055e0d9bbd..89a6302351a54 100644 --- a/x-pack/plugins/endpoint/public/applications/endpoint/index.tsx +++ b/x-pack/plugins/endpoint/public/applications/endpoint/index.tsx @@ -7,13 +7,9 @@ import * as React from 'react'; import ReactDOM from 'react-dom'; import { CoreStart, AppMountParameters, ScopedHistory } from 'kibana/public'; -import { I18nProvider, FormattedMessage } from '@kbn/i18n/react'; -import { Route, Switch, Router } from 'react-router-dom'; -import { Provider } from 'react-redux'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { Route, Switch } from 'react-router-dom'; import { Store } from 'redux'; -import { useObservable } from 'react-use'; -import { KibanaContextProvider } from '../../../../../../src/plugins/kibana_react/public'; -import { RouteCapture } from './view/route_capture'; import { EndpointPluginStartDependencies } from '../../plugin'; import { appStoreFactory } from './store'; import { AlertIndex } from './view/alerts'; @@ -21,7 +17,7 @@ import { HostList } from './view/hosts'; import { PolicyList } from './view/policy'; import { PolicyDetails } from './view/policy'; import { HeaderNavigation } from './components/header_nav'; -import { EuiThemeProvider } from '../../../../../legacy/common/eui_styled_components'; +import { AppRootProvider } from './view/app_root_provider'; /** * This module will be loaded asynchronously to reduce the bundle size of your plugin's main bundle. @@ -49,54 +45,31 @@ interface RouterProps { } const AppRoot: React.FunctionComponent = React.memo( - ({ - history, - store, - coreStart: { http, notifications, uiSettings, application }, - depsStart: { data }, - }) => { - const isDarkMode = useObservable(uiSettings.get$('theme:darkMode')); - + ({ history, store, coreStart, depsStart }) => { return ( - - - - - - - - - ( -

- -

- )} - /> - - - - - ( - - )} - /> - - - - - - - + + + + ( +

+ +

+ )} + /> + + + + + ( + + )} + /> + + ); } ); diff --git a/x-pack/plugins/endpoint/public/applications/endpoint/mocks/app_context_render.tsx b/x-pack/plugins/endpoint/public/applications/endpoint/mocks/app_context_render.tsx new file mode 100644 index 0000000000000..af34205e2310f --- /dev/null +++ b/x-pack/plugins/endpoint/public/applications/endpoint/mocks/app_context_render.tsx @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import { createMemoryHistory } from 'history'; +import { render as reactRender, RenderOptions, RenderResult } from '@testing-library/react'; +import { appStoreFactory } from '../store'; +import { coreMock } from '../../../../../../../src/core/public/mocks'; +import { EndpointPluginStartDependencies } from '../../../plugin'; +import { depsStartMock } from './dependencies_start_mock'; +import { AppRootProvider } from '../view/app_root_provider'; + +type UiRender = (ui: React.ReactElement, options?: RenderOptions) => RenderResult; + +/** + * Mocked app root context renderer + */ +interface AppContextTestRender { + store: ReturnType; + history: ReturnType; + coreStart: ReturnType; + depsStart: EndpointPluginStartDependencies; + /** + * A wrapper around `AppRootContext` component. Uses the mocked modules as input to the + * `AppRootContext` + */ + AppWrapper: React.FC; + /** + * Renders the given UI within the created `AppWrapper` providing the given UI a mocked + * endpoint runtime context environment + */ + render: UiRender; +} + +/** + * Creates a mocked endpoint app context custom renderer that can be used to render + * component that depend upon the application's surrounding context providers. + * Factory also returns the content that was used to create the custom renderer, allowing + * for further customization. + */ +export const createAppRootMockRenderer = (): AppContextTestRender => { + const history = createMemoryHistory(); + const coreStart = coreMock.createStart({ basePath: '/mock' }); + const depsStart = depsStartMock(); + const store = appStoreFactory({ coreStart, depsStart }); + const AppWrapper: React.FunctionComponent<{ children: React.ReactElement }> = ({ children }) => ( + + {children} + + ); + const render: UiRender = (ui, options) => { + // @ts-ignore + return reactRender(ui, { + wrapper: AppWrapper, + ...options, + }); + }; + + return { + store, + history, + coreStart, + depsStart, + AppWrapper, + render, + }; +}; diff --git a/x-pack/plugins/endpoint/public/applications/endpoint/mocks.ts b/x-pack/plugins/endpoint/public/applications/endpoint/mocks/dependencies_start_mock.ts similarity index 96% rename from x-pack/plugins/endpoint/public/applications/endpoint/mocks.ts rename to x-pack/plugins/endpoint/public/applications/endpoint/mocks/dependencies_start_mock.ts index e1a90b4a416dc..00cf0bca57e66 100644 --- a/x-pack/plugins/endpoint/public/applications/endpoint/mocks.ts +++ b/x-pack/plugins/endpoint/public/applications/endpoint/mocks/dependencies_start_mock.ts @@ -7,7 +7,7 @@ import { dataPluginMock, Start as DataPublicStartMock, -} from '../../../../../../src/plugins/data/public/mocks'; +} from '../../../../../../../src/plugins/data/public/mocks'; type DataMock = Omit & { indexPatterns: Omit & { diff --git a/x-pack/plugins/endpoint/public/applications/endpoint/mocks/index.ts b/x-pack/plugins/endpoint/public/applications/endpoint/mocks/index.ts new file mode 100644 index 0000000000000..65e78f27943ba --- /dev/null +++ b/x-pack/plugins/endpoint/public/applications/endpoint/mocks/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './dependencies_start_mock'; +export * from './app_context_render'; diff --git a/x-pack/plugins/endpoint/public/applications/endpoint/store/policy_details/reducer.ts b/x-pack/plugins/endpoint/public/applications/endpoint/store/policy_details/reducer.ts index af9b49cea18ad..fb3e26157ef32 100644 --- a/x-pack/plugins/endpoint/public/applications/endpoint/store/policy_details/reducer.ts +++ b/x-pack/plugins/endpoint/public/applications/endpoint/store/policy_details/reducer.ts @@ -74,8 +74,12 @@ export const policyDetailsReducer: Reducer = ( ...state, location: action.payload, }; + const isCurrentlyOnDetailsPage = isOnPolicyDetailsPage(newState); + const wasPreviouslyOnDetailsPage = isOnPolicyDetailsPage(state); - if (isOnPolicyDetailsPage(newState)) { + // Did user just enter the Detail page? if so, then set the loading indicator and return new state + if (isCurrentlyOnDetailsPage && !wasPreviouslyOnDetailsPage) { + newState.isLoading = true; return newState; } return { diff --git a/x-pack/plugins/endpoint/public/applications/endpoint/view/app_root_provider.tsx b/x-pack/plugins/endpoint/public/applications/endpoint/view/app_root_provider.tsx new file mode 100644 index 0000000000000..ca27831ee90b5 --- /dev/null +++ b/x-pack/plugins/endpoint/public/applications/endpoint/view/app_root_provider.tsx @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { memo, ReactNode, useMemo } from 'react'; +import { Provider } from 'react-redux'; +import { I18nProvider } from '@kbn/i18n/react'; +import { Router } from 'react-router-dom'; +import { History } from 'history'; +import { CoreStart } from 'kibana/public'; +import { useObservable } from 'react-use'; +import { EuiThemeProvider } from '../../../../../../legacy/common/eui_styled_components'; +import { KibanaContextProvider } from '../../../../../../../src/plugins/kibana_react/public'; +import { appStoreFactory } from '../store'; +import { RouteCapture } from './route_capture'; +import { EndpointPluginStartDependencies } from '../../../plugin'; + +/** + * Provides the context for rendering the endpoint app + */ +export const AppRootProvider = memo<{ + store: ReturnType; + history: History; + coreStart: CoreStart; + depsStart: EndpointPluginStartDependencies; + children: ReactNode | ReactNode[]; +}>( + ({ + store, + history, + coreStart: { http, notifications, uiSettings, application }, + depsStart: { data }, + children, + }) => { + const isDarkMode = useObservable(uiSettings.get$('theme:darkMode')); + const services = useMemo(() => ({ http, notifications, application, data }), [ + application, + data, + http, + notifications, + ]); + return ( + + + + + + {children} + + + + + + ); + } +); diff --git a/x-pack/plugins/endpoint/public/applications/endpoint/view/policy/agents_summary.tsx b/x-pack/plugins/endpoint/public/applications/endpoint/view/policy/agents_summary.tsx index d0751cf9fb886..a3e30eb891db4 100644 --- a/x-pack/plugins/endpoint/public/applications/endpoint/view/policy/agents_summary.tsx +++ b/x-pack/plugins/endpoint/public/applications/endpoint/view/policy/agents_summary.tsx @@ -61,7 +61,7 @@ export const AgentsSummary = memo(props => { }, []); return ( - + {stats.map(({ key, title, health }) => { return ( diff --git a/x-pack/plugins/endpoint/public/applications/endpoint/view/policy/policy_details.test.tsx b/x-pack/plugins/endpoint/public/applications/endpoint/view/policy/policy_details.test.tsx new file mode 100644 index 0000000000000..2ecc2b117bf01 --- /dev/null +++ b/x-pack/plugins/endpoint/public/applications/endpoint/view/policy/policy_details.test.tsx @@ -0,0 +1,239 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import { mount } from 'enzyme'; +import { createAppRootMockRenderer } from '../../mocks'; +import { PolicyDetails } from './policy_details'; +import { EndpointDocGenerator } from '../../../../../common/generate_data'; + +describe('Policy Details', () => { + type FindReactWrapperResponse = ReturnType['find']>; + + const sleep = (ms = 100) => new Promise(wakeup => setTimeout(wakeup, ms)); + const generator = new EndpointDocGenerator(); + const { history, AppWrapper, coreStart } = createAppRootMockRenderer(); + const http = coreStart.http; + const render = (ui: Parameters[0]) => mount(ui, { wrappingComponent: AppWrapper }); + let policyDatasource: ReturnType; + let policyView: ReturnType; + + beforeEach(() => jest.clearAllMocks()); + + afterEach(() => { + if (policyView) { + policyView.unmount(); + } + }); + + describe('when displayed with invalid id', () => { + beforeEach(() => { + http.get.mockReturnValue(Promise.reject(new Error('policy not found'))); + history.push('/policy/1'); + policyView = render(); + }); + + it('should show loader followed by error message', () => { + expect(policyView.find('EuiLoadingSpinner').length).toBe(1); + policyView.update(); + const callout = policyView.find('EuiCallOut'); + expect(callout).toHaveLength(1); + expect(callout.prop('color')).toEqual('danger'); + expect(callout.text()).toEqual('policy not found'); + }); + }); + describe('when displayed with valid id', () => { + let asyncActions: Promise = Promise.resolve(); + + beforeEach(() => { + policyDatasource = generator.generatePolicyDatasource(); + policyDatasource.id = '1'; + + http.get.mockImplementation((...args) => { + const [path] = args; + if (typeof path === 'string') { + // GET datasouce + if (path === '/api/ingest_manager/datasources/1') { + asyncActions = asyncActions.then(async (): Promise => await sleep()); + return Promise.resolve({ + item: policyDatasource, + success: true, + }); + } + + // GET Agent status for agent config + if (path === '/api/ingest_manager/fleet/agent-status') { + asyncActions = asyncActions.then(async () => await sleep()); + return Promise.resolve({ + results: { events: 0, total: 5, online: 3, error: 1, offline: 1 }, + success: true, + }); + } + } + + return Promise.reject(new Error('unknown API call!')); + }); + history.push('/policy/1'); + policyView = render(); + }); + + it('should display back to list button and policy title', () => { + policyView.update(); + const pageHeaderLeft = policyView.find( + 'EuiPageHeaderSection[data-test-subj="pageViewHeaderLeft"]' + ); + + const backToListButton = pageHeaderLeft.find('EuiButtonEmpty'); + expect(backToListButton.prop('iconType')).toBe('arrowLeft'); + expect(backToListButton.prop('href')).toBe('/mock/app/endpoint/policy'); + expect(backToListButton.text()).toBe('Back to policy list'); + + const pageTitle = pageHeaderLeft.find('[data-test-subj="pageViewHeaderLeftTitle"]'); + expect(pageTitle).toHaveLength(1); + expect(pageTitle.text()).toEqual(policyDatasource.name); + }); + it('should navigate to list if back to link is clicked', async () => { + policyView.update(); + const backToListButton = policyView.find( + 'EuiPageHeaderSection[data-test-subj="pageViewHeaderLeft"] EuiButtonEmpty' + ); + expect(history.location.pathname).toEqual('/policy/1'); + backToListButton.simulate('click'); + expect(history.location.pathname).toEqual('/policy'); + }); + it('should display agent stats', async () => { + await asyncActions; + policyView.update(); + const headerRight = policyView.find( + 'EuiPageHeaderSection[data-test-subj="pageViewHeaderRight"]' + ); + const agentsSummary = headerRight.find('EuiFlexGroup[data-test-subj="policyAgentsSummary"]'); + expect(agentsSummary).toHaveLength(1); + expect(agentsSummary.text()).toBe('Hosts5Online3Offline1Error1'); + }); + it('should display cancel button', async () => { + await asyncActions; + policyView.update(); + const cancelbutton = policyView.find( + 'EuiButtonEmpty[data-test-subj="policyDetailsCancelButton"]' + ); + expect(cancelbutton).toHaveLength(1); + expect(cancelbutton.text()).toEqual('Cancel'); + }); + it('should redirect to policy list when cancel button is clicked', async () => { + await asyncActions; + policyView.update(); + const cancelbutton = policyView.find( + 'EuiButtonEmpty[data-test-subj="policyDetailsCancelButton"]' + ); + expect(history.location.pathname).toEqual('/policy/1'); + cancelbutton.simulate('click'); + expect(history.location.pathname).toEqual('/policy'); + }); + it('should display save button', async () => { + await asyncActions; + policyView.update(); + const saveButton = policyView.find('EuiButton[data-test-subj="policyDetailsSaveButton"]'); + expect(saveButton).toHaveLength(1); + expect(saveButton.text()).toEqual('Save'); + }); + describe('when the save button is clicked', () => { + let saveButton: FindReactWrapperResponse; + let confirmModal: FindReactWrapperResponse; + let modalCancelButton: FindReactWrapperResponse; + let modalConfirmButton: FindReactWrapperResponse; + + beforeEach(async () => { + await asyncActions; + policyView.update(); + saveButton = policyView.find('EuiButton[data-test-subj="policyDetailsSaveButton"]'); + saveButton.simulate('click'); + policyView.update(); + confirmModal = policyView.find( + 'EuiConfirmModal[data-test-subj="policyDetailsConfirmModal"]' + ); + modalCancelButton = confirmModal.find('button[data-test-subj="confirmModalCancelButton"]'); + modalConfirmButton = confirmModal.find( + 'button[data-test-subj="confirmModalConfirmButton"]' + ); + http.put.mockImplementation((...args) => { + asyncActions = asyncActions.then(async () => await sleep()); + const [path] = args; + if (typeof path === 'string') { + if (path === '/api/ingest_manager/datasources/1') { + return Promise.resolve({ + item: policyDatasource, + success: true, + }); + } + } + + return Promise.reject(new Error('unknown PUT path!')); + }); + }); + + it('should show a modal confirmation', () => { + expect(confirmModal).toHaveLength(1); + expect(confirmModal.find('div[data-test-subj="confirmModalTitleText"]').text()).toEqual( + 'Save and deploy changes' + ); + expect(modalCancelButton.text()).toEqual('Cancel'); + expect(modalConfirmButton.text()).toEqual('Save and deploy changes'); + }); + it('should show info callout if policy is in use', () => { + const warningCallout = confirmModal.find( + 'EuiCallOut[data-test-subj="policyDetailsWarningCallout"]' + ); + expect(warningCallout).toHaveLength(1); + expect(warningCallout.text()).toEqual( + 'This action will update 5 hostsSaving these changes will apply the updates to all active endpoints assigned to this policy' + ); + }); + it('should close dialog if cancel button is clicked', () => { + modalCancelButton.simulate('click'); + expect( + policyView.find('EuiConfirmModal[data-test-subj="policyDetailsConfirmModal"]') + ).toHaveLength(0); + }); + it('should update policy and show success notification when confirm button is clicked', async () => { + modalConfirmButton.simulate('click'); + policyView.update(); + // Modal should be closed + expect( + policyView.find('EuiConfirmModal[data-test-subj="policyDetailsConfirmModal"]') + ).toHaveLength(0); + + // API should be called + await asyncActions; + expect(http.put.mock.calls[0][0]).toEqual(`/api/ingest_manager/datasources/1`); + policyView.update(); + + // Toast notification should be shown + const toastAddMock = coreStart.notifications.toasts.add.mock; + expect(toastAddMock.calls).toHaveLength(1); + expect(toastAddMock.calls[0][0]).toMatchObject({ + color: 'success', + iconType: 'check', + }); + }); + it('should show an error notification toast if update fails', async () => { + policyDatasource.id = 'invalid'; + modalConfirmButton.simulate('click'); + + await asyncActions; + policyView.update(); + + // Toast notification should be shown + const toastAddMock = coreStart.notifications.toasts.add.mock; + expect(toastAddMock.calls).toHaveLength(1); + expect(toastAddMock.calls[0][0]).toMatchObject({ + color: 'danger', + iconType: 'alert', + }); + }); + }); + }); +}); diff --git a/x-pack/plugins/endpoint/public/applications/endpoint/view/policy/policy_details.tsx b/x-pack/plugins/endpoint/public/applications/endpoint/view/policy/policy_details.tsx index 2dba301bf4537..bc56e5e6f6329 100644 --- a/x-pack/plugins/endpoint/public/applications/endpoint/view/policy/policy_details.tsx +++ b/x-pack/plugins/endpoint/public/applications/endpoint/view/policy/policy_details.tsx @@ -149,7 +149,10 @@ export const PolicyDetails = React.memo(() => { - + @@ -157,6 +160,7 @@ export const PolicyDetails = React.memo(() => { 0 && ( <> ; series?: MetricsExplorerSeries; @@ -57,14 +52,35 @@ interface Props { criteria: MetricExpression[]; groupBy?: string; filterQuery?: string; + sourceId?: string; }; alertsContext: AlertsContextValue; setAlertParams(key: string, value: any): void; setAlertProperty(key: string, value: any): void; } -type Comparator = '>' | '>=' | 'between' | '<' | '<='; type TimeUnit = 's' | 'm' | 'h' | 'd'; +type MetricExpression = Omit & { + metric?: string; +}; + +enum AGGREGATION_TYPES { + COUNT = 'count', + AVERAGE = 'avg', + SUM = 'sum', + MIN = 'min', + MAX = 'max', + RATE = 'rate', + CARDINALITY = 'cardinality', +} + +const defaultExpression = { + aggType: AGGREGATION_TYPES.AVERAGE, + comparator: Comparator.GT, + threshold: [], + timeSize: 1, + timeUnit: 'm', +} as MetricExpression; export const Expressions: React.FC = props => { const { setAlertParams, alertParams, errors, alertsContext } = props; @@ -87,18 +103,6 @@ export const Expressions: React.FC = props => { } }, [alertsContext.metadata]); - const defaultExpression = useMemo( - () => ({ - aggType: AGGREGATION_TYPES.AVERAGE, - comparator: '>', - threshold: [], - timeSize: 1, - timeUnit: 'm', - indexPattern: source?.configuration.metricAlias, - }), - [source] - ); - const updateParams = useCallback( (id, e: MetricExpression) => { const exp = alertParams.criteria ? alertParams.criteria.slice() : []; @@ -112,7 +116,7 @@ export const Expressions: React.FC = props => { const exp = alertParams.criteria.slice(); exp.push(defaultExpression); setAlertParams('criteria', exp); - }, [setAlertParams, alertParams.criteria, defaultExpression]); + }, [setAlertParams, alertParams.criteria]); const removeExpression = useCallback( (id: number) => { @@ -179,11 +183,10 @@ export const Expressions: React.FC = props => { 'criteria', md.currentOptions.metrics.map(metric => ({ metric: metric.field, - comparator: '>', + comparator: Comparator.GT, threshold: [], timeSize, timeUnit, - indexPattern: source?.configuration.metricAlias, aggType: metric.aggregation, })) ); @@ -201,6 +204,7 @@ export const Expressions: React.FC = props => { setAlertParams('groupBy', md.currentOptions.groupBy); } + setAlertParams('sourceId', source?.id); } }, [alertsContext.metadata, defaultExpression, source]); // eslint-disable-line react-hooks/exhaustive-deps @@ -325,17 +329,22 @@ const StyledExpression = euiStyled.div` export const ExpressionRow: React.FC = props => { const { setAlertParams, expression, errors, expressionId, remove, fields, canDelete } = props; - const { aggType = AGGREGATION_TYPES.MAX, metric, comparator = '>', threshold = [] } = expression; + const { + aggType = AGGREGATION_TYPES.MAX, + metric, + comparator = Comparator.GT, + threshold = [], + } = expression; const updateAggType = useCallback( (at: string) => { - setAlertParams(expressionId, { ...expression, aggType: at }); + setAlertParams(expressionId, { ...expression, aggType: at as MetricExpression['aggType'] }); }, [expressionId, expression, setAlertParams] ); const updateMetric = useCallback( - (m?: string) => { + (m?: MetricExpression['metric']) => { setAlertParams(expressionId, { ...expression, metric: m }); }, [expressionId, expression, setAlertParams] @@ -384,7 +393,7 @@ export const ExpressionRow: React.FC = props => { )} '} + thresholdComparator={comparator || Comparator.GT} threshold={threshold} onChangeSelectedThresholdComparator={updateComparator} onChangeSelectedThreshold={updateThreshold} @@ -411,16 +420,6 @@ export const ExpressionRow: React.FC = props => { ); }; -enum AGGREGATION_TYPES { - COUNT = 'count', - AVERAGE = 'avg', - SUM = 'sum', - MIN = 'min', - MAX = 'max', - RATE = 'rate', - CARDINALITY = 'cardinality', -} - export const aggregationType: { [key: string]: any } = { avg: { text: i18n.translate('xpack.infra.metrics.alertFlyout.aggregationText.avg', { diff --git a/x-pack/plugins/infra/public/components/alerting/metrics/validation.tsx b/x-pack/plugins/infra/public/components/alerting/metrics/validation.tsx index 0f5b07f8c0e13..55365c3d4c2ba 100644 --- a/x-pack/plugins/infra/public/components/alerting/metrics/validation.tsx +++ b/x-pack/plugins/infra/public/components/alerting/metrics/validation.tsx @@ -6,15 +6,14 @@ import { i18n } from '@kbn/i18n'; // eslint-disable-next-line @kbn/eslint/no-restricted-paths - -import { MetricExpression } from './expression'; +import { MetricExpressionParams } from '../../../../server/lib/alerting/metric_threshold/types'; // eslint-disable-next-line @kbn/eslint/no-restricted-paths import { ValidationResult } from '../../../../../triggers_actions_ui/public/types'; export function validateMetricThreshold({ criteria, }: { - criteria: MetricExpression[]; + criteria: MetricExpressionParams[]; }): ValidationResult { const validationResult = { errors: {} }; const errors: { diff --git a/x-pack/plugins/infra/server/lib/alerting/metric_threshold/metric_threshold_executor.test.ts b/x-pack/plugins/infra/server/lib/alerting/metric_threshold/metric_threshold_executor.test.ts index 09f1702349542..38cd0cec145f9 100644 --- a/x-pack/plugins/infra/server/lib/alerting/metric_threshold/metric_threshold_executor.test.ts +++ b/x-pack/plugins/infra/server/lib/alerting/metric_threshold/metric_threshold_executor.test.ts @@ -16,7 +16,8 @@ const executor = createMetricThresholdExecutor('test') as (opts: { const alertInstances = new Map(); const services = { - callCluster(_: string, { body }: any) { + callCluster(_: string, { body, index }: any) { + if (index === 'alternatebeat-*') return mocks.changedSourceIdResponse; const metric = body.query.bool.filter[1]?.exists.field; if (body.aggs.groupings) { if (body.aggs.groupings.composite.after) { @@ -55,6 +56,13 @@ const services = { }, }; }, + savedObjectsClient: { + get(_: string, sourceId: string) { + if (sourceId === 'alternate') + return { id: 'alternate', attributes: { metricAlias: 'alternatebeat-*' } }; + return { id: 'default', attributes: { metricAlias: 'metricbeat-*' } }; + }, + }, }; const baseCriterion = { @@ -62,15 +70,15 @@ const baseCriterion = { metric: 'test.metric.1', timeSize: 1, timeUnit: 'm', - indexPattern: 'metricbeat-*', }; describe('The metric threshold alert type', () => { describe('querying the entire infrastructure', () => { const instanceID = 'test-*'; - const execute = (comparator: Comparator, threshold: number[]) => + const execute = (comparator: Comparator, threshold: number[], sourceId: string = 'default') => executor({ services, params: { + sourceId, criteria: [ { ...baseCriterion, @@ -134,6 +142,14 @@ describe('The metric threshold alert type', () => { expect(mostRecentAction.action.thresholdOf.condition0).toStrictEqual([0.75]); expect(mostRecentAction.action.metricOf.condition0).toBe('test.metric.1'); }); + test('fetches the index pattern dynamically', async () => { + await execute(Comparator.LT, [17], 'alternate'); + expect(alertInstances.get(instanceID).mostRecentAction.id).toBe(FIRED_ACTIONS.id); + expect(alertInstances.get(instanceID).state.alertState).toBe(AlertStates.ALERT); + await execute(Comparator.LT, [1.5], 'alternate'); + expect(alertInstances.get(instanceID).mostRecentAction).toBe(undefined); + expect(alertInstances.get(instanceID).state.alertState).toBe(AlertStates.OK); + }); }); describe('querying with a groupBy parameter', () => { diff --git a/x-pack/plugins/infra/server/lib/alerting/metric_threshold/metric_threshold_executor.ts b/x-pack/plugins/infra/server/lib/alerting/metric_threshold/metric_threshold_executor.ts index 60bba61b75ef1..c5ea65f7a4d1a 100644 --- a/x-pack/plugins/infra/server/lib/alerting/metric_threshold/metric_threshold_executor.ts +++ b/x-pack/plugins/infra/server/lib/alerting/metric_threshold/metric_threshold_executor.ts @@ -5,6 +5,8 @@ */ import { mapValues } from 'lodash'; import { i18n } from '@kbn/i18n'; +import { convertSavedObjectToSavedSourceConfiguration } from '../../sources/sources'; +import { infraSourceConfigurationSavedObjectType } from '../../sources/saved_object_mappings'; import { InfraDatabaseSearchResponse } from '../../adapters/framework/adapter_types'; import { createAfterKeyHandler } from '../../../utils/create_afterkey_handler'; import { getAllCompositeData } from '../../../utils/get_all_composite_data'; @@ -15,6 +17,7 @@ import { getIntervalInSeconds } from '../../../utils/get_interval_in_seconds'; import { getDateHistogramOffset } from '../../snapshot/query_helpers'; const TOTAL_BUCKETS = 5; +const DEFAULT_INDEX_PATTERN = 'metricbeat-*'; interface Aggregation { aggregatedIntervals: { @@ -165,18 +168,42 @@ export const getElasticsearchMetricQuery = ( }; }; +const getIndexPattern: ( + services: AlertServices, + sourceId?: string +) => Promise = async function({ savedObjectsClient }, sourceId = 'default') { + try { + const sourceConfiguration = await savedObjectsClient.get( + infraSourceConfigurationSavedObjectType, + sourceId + ); + const { metricAlias } = convertSavedObjectToSavedSourceConfiguration( + sourceConfiguration + ).configuration; + return metricAlias || DEFAULT_INDEX_PATTERN; + } catch (e) { + if (e.output.statusCode === 404) { + return DEFAULT_INDEX_PATTERN; + } else { + throw e; + } + } +}; + const getMetric: ( services: AlertServices, params: MetricExpressionParams, + index: string, groupBy: string | undefined, filterQuery: string | undefined ) => Promise> = async function( - { callCluster }, + { savedObjectsClient, callCluster }, params, + index, groupBy, filterQuery ) { - const { indexPattern, aggType } = params; + const { aggType } = params; const searchBody = getElasticsearchMetricQuery(params, groupBy, filterQuery); try { @@ -189,7 +216,7 @@ const getMetric: ( response => response.aggregations?.groupings?.after_key ); const compositeBuckets = (await getAllCompositeData( - body => callCluster('search', { body, index: indexPattern }), + body => callCluster('search', { body, index }), searchBody, bucketSelector, afterKeyHandler @@ -204,7 +231,7 @@ const getMetric: ( } const result = await callCluster('search', { body: searchBody, - index: indexPattern, + index, }); return { '*': getCurrentValueFromAggregations(result.aggregations, aggType) }; } catch (e) { @@ -236,16 +263,18 @@ const mapToConditionsLookup = ( export const createMetricThresholdExecutor = (alertUUID: string) => async function({ services, params }: AlertExecutorOptions) { - const { criteria, groupBy, filterQuery } = params as { + const { criteria, groupBy, filterQuery, sourceId } = params as { criteria: MetricExpressionParams[]; groupBy: string | undefined; filterQuery: string | undefined; + sourceId?: string; }; const alertResults = await Promise.all( criteria.map(criterion => (async () => { - const currentValues = await getMetric(services, criterion, groupBy, filterQuery); + const index = await getIndexPattern(services, sourceId); + const currentValues = await getMetric(services, criterion, index, groupBy, filterQuery); const { threshold, comparator } = criterion; const comparisonFunction = comparatorMap[comparator]; return mapValues(currentValues, value => ({ diff --git a/x-pack/plugins/infra/server/lib/alerting/metric_threshold/register_metric_threshold_alert_type.ts b/x-pack/plugins/infra/server/lib/alerting/metric_threshold/register_metric_threshold_alert_type.ts index 2ab3a3322661d..8808219cabaa7 100644 --- a/x-pack/plugins/infra/server/lib/alerting/metric_threshold/register_metric_threshold_alert_type.ts +++ b/x-pack/plugins/infra/server/lib/alerting/metric_threshold/register_metric_threshold_alert_type.ts @@ -29,7 +29,6 @@ export async function registerMetricThresholdAlertType(alertingPlugin: PluginSet ]), timeUnit: schema.string(), timeSize: schema.number(), - indexPattern: schema.string(), }; const nonCountCriterion = schema.object({ @@ -89,6 +88,7 @@ export async function registerMetricThresholdAlertType(alertingPlugin: PluginSet criteria: schema.arrayOf(schema.oneOf([countCriterion, nonCountCriterion])), groupBy: schema.maybe(schema.string()), filterQuery: schema.maybe(schema.string()), + sourceId: schema.string(), }), }, defaultActionGroupId: FIRED_ACTIONS.id, diff --git a/x-pack/plugins/infra/server/lib/alerting/metric_threshold/test_mocks.ts b/x-pack/plugins/infra/server/lib/alerting/metric_threshold/test_mocks.ts index e87ffcfb2b912..66e0a363c8983 100644 --- a/x-pack/plugins/infra/server/lib/alerting/metric_threshold/test_mocks.ts +++ b/x-pack/plugins/infra/server/lib/alerting/metric_threshold/test_mocks.ts @@ -26,6 +26,17 @@ const bucketsB = [ }, ]; +const bucketsC = [ + { + doc_count: 2, + aggregatedValue: { value: 0.5 }, + }, + { + doc_count: 3, + aggregatedValue: { value: 16.0 }, + }, +]; + export const basicMetricResponse = { aggregations: { aggregatedIntervals: { @@ -108,3 +119,11 @@ export const compositeEndResponse = { aggregations: {}, hits: { total: { value: 0 } }, }; + +export const changedSourceIdResponse = { + aggregations: { + aggregatedIntervals: { + buckets: bucketsC, + }, + }, +}; diff --git a/x-pack/plugins/infra/server/lib/alerting/metric_threshold/types.ts b/x-pack/plugins/infra/server/lib/alerting/metric_threshold/types.ts index 557a071ec9175..abed691f109c0 100644 --- a/x-pack/plugins/infra/server/lib/alerting/metric_threshold/types.ts +++ b/x-pack/plugins/infra/server/lib/alerting/metric_threshold/types.ts @@ -28,7 +28,7 @@ export type TimeUnit = 's' | 'm' | 'h' | 'd'; interface BaseMetricExpressionParams { timeSize: number; timeUnit: TimeUnit; - indexPattern: string; + sourceId?: string; threshold: number[]; comparator: Comparator; } diff --git a/x-pack/plugins/infra/server/lib/sources/sources.ts b/x-pack/plugins/infra/server/lib/sources/sources.ts index 5b9207a88e6e4..c7ff6c9638204 100644 --- a/x-pack/plugins/infra/server/lib/sources/sources.ts +++ b/x-pack/plugins/infra/server/lib/sources/sources.ts @@ -231,7 +231,7 @@ const mergeSourceConfiguration = ( first ); -const convertSavedObjectToSavedSourceConfiguration = (savedObject: unknown) => +export const convertSavedObjectToSavedSourceConfiguration = (savedObject: unknown) => pipe( SourceConfigurationSavedObjectRuntimeType.decode(savedObject), map(savedSourceConfiguration => ({ diff --git a/x-pack/plugins/ingest_manager/common/services/package_to_config.test.ts b/x-pack/plugins/ingest_manager/common/services/package_to_config.test.ts index 357f407811880..5fa7af2dda79a 100644 --- a/x-pack/plugins/ingest_manager/common/services/package_to_config.test.ts +++ b/x-pack/plugins/ingest_manager/common/services/package_to_config.test.ts @@ -24,6 +24,7 @@ describe('Ingest Manager - packageToConfig', () => { visualization: [], search: [], 'index-pattern': [], + map: [], }, }, status: InstallationStatus.notInstalled, diff --git a/x-pack/plugins/ingest_manager/common/types/models/epm.ts b/x-pack/plugins/ingest_manager/common/types/models/epm.ts index 28786530db018..efa6621001038 100644 --- a/x-pack/plugins/ingest_manager/common/types/models/epm.ts +++ b/x-pack/plugins/ingest_manager/common/types/models/epm.ts @@ -28,6 +28,7 @@ export enum KibanaAssetType { visualization = 'visualization', search = 'search', indexPattern = 'index-pattern', + map = 'map', } export enum ElasticsearchAssetType { diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/epm/constants.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/epm/constants.tsx index 3a6dfe4a87daf..685199245df18 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/epm/constants.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/epm/constants.tsx @@ -24,6 +24,7 @@ export const AssetTitleMap: Record = { search: 'Saved Search', visualization: 'Visualization', input: 'Agent input', + map: 'Map', }; export const ServiceTitleMap: Record = { @@ -36,6 +37,7 @@ export const AssetIcons: Record = { 'index-pattern': 'indexPatternApp', search: 'searchProfilerApp', visualization: 'visualizeApp', + map: 'mapApp', }; export const ServiceIcons: Record = { diff --git a/x-pack/plugins/ml/common/types/errors.ts b/x-pack/plugins/ml/common/types/errors.ts index 63e222490082b..284250bd5ce55 100644 --- a/x-pack/plugins/ml/common/types/errors.ts +++ b/x-pack/plugins/ml/common/types/errors.ts @@ -9,6 +9,7 @@ export interface ErrorResponse { statusCode: number; error: string; message: string; + attributes?: any; }; name: string; } diff --git a/x-pack/plugins/ml/common/types/file_datavisualizer.ts b/x-pack/plugins/ml/common/types/file_datavisualizer.ts index bc03f82673a1f..f771547b97811 100644 --- a/x-pack/plugins/ml/common/types/file_datavisualizer.ts +++ b/x-pack/plugins/ml/common/types/file_datavisualizer.ts @@ -4,6 +4,21 @@ * you may not use this file except in compliance with the Elastic License. */ +export interface InputOverrides { + [key: string]: string; +} + +export type FormattedOverrides = InputOverrides & { + column_names: string[]; + has_header_row: boolean; + should_trim_fields: boolean; +}; + +export interface AnalysisResult { + results: FindFileStructureResponse; + overrides?: FormattedOverrides; +} + export interface FindFileStructureResponse { charset: string; has_header_row: boolean; @@ -28,4 +43,54 @@ export interface FindFileStructureResponse { need_client_timezone: boolean; num_lines_analyzed: number; column_names: string[]; + explanation?: string[]; + grok_pattern?: string; + multiline_start_pattern?: string; + exclude_lines_pattern?: string; + java_timestamp_formats?: string[]; + joda_timestamp_formats?: string[]; + timestamp_field?: string; + should_trim_fields?: boolean; +} + +export interface ImportResponse { + success: boolean; + id: string; + index?: string; + pipelineId?: string; + docCount: number; + failures: ImportFailure[]; + error?: any; + ingestError?: boolean; +} + +export interface ImportFailure { + item: number; + reason: string; + doc: Doc; +} + +export interface Doc { + message: string; +} + +export interface Settings { + pipeline?: string; + index: string; + body: any[]; + [key: string]: any; +} + +export interface Mappings { + [key: string]: any; +} + +export interface IngestPipelineWrapper { + id: string; + pipeline: IngestPipeline; +} + +export interface IngestPipeline { + description: string; + processors: any[]; } diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/about_panel/about_panel.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/about_panel/about_panel.tsx similarity index 91% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/about_panel/about_panel.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/about_panel/about_panel.tsx index edecc925591d3..2bddf0de0499d 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/about_panel/about_panel.js +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/about_panel/about_panel.tsx @@ -6,7 +6,7 @@ import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; -import React from 'react'; +import React, { FC } from 'react'; import { EuiFlexGroup, @@ -23,7 +23,11 @@ import { import { WelcomeContent } from './welcome_content'; -export function AboutPanel({ onFilePickerChange }) { +interface Props { + onFilePickerChange(files: FileList | null): void; +} + +export const AboutPanel: FC = ({ onFilePickerChange }) => { return ( @@ -54,9 +58,9 @@ export function AboutPanel({ onFilePickerChange }) { ); -} +}; -export function LoadingPanel() { +export const LoadingPanel: FC = () => { return ( @@ -79,4 +83,4 @@ export function LoadingPanel() { ); -} +}; diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/about_panel/index.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/about_panel/index.ts similarity index 100% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/about_panel/index.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/about_panel/index.ts diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/about_panel/welcome_content.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/about_panel/welcome_content.tsx similarity index 90% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/about_panel/welcome_content.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/about_panel/welcome_content.tsx index 73d12122879f8..c56ab021e2f2f 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/about_panel/welcome_content.js +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/about_panel/welcome_content.tsx @@ -5,7 +5,8 @@ */ import { FormattedMessage } from '@kbn/i18n/react'; -import React from 'react'; +import React, { FC } from 'react'; +import { i18n } from '@kbn/i18n'; import { EuiFlexGroup, @@ -19,7 +20,14 @@ import { import { ExperimentalBadge } from '../experimental_badge'; -export function WelcomeContent() { +export const WelcomeContent: FC = () => { + const toolTipContent = i18n.translate( + 'xpack.ml.fileDatavisualizer.welcomeContent.experimentalFeatureTooltip', + { + defaultMessage: "Experimental feature. We'd love to hear your feedback.", + } + ); + return ( @@ -32,16 +40,7 @@ export function WelcomeContent() { id="xpack.ml.fileDatavisualizer.welcomeContent.visualizeDataFromLogFileTitle" defaultMessage="Visualize data from a log file {experimentalBadge}" values={{ - experimentalBadge: ( - - } - /> - ), + experimentalBadge: , }} /> @@ -144,4 +143,4 @@ export function WelcomeContent() { ); -} +}; diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/analysis_summary/analysis_summary.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/analysis_summary/analysis_summary.tsx similarity index 89% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/analysis_summary/analysis_summary.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/analysis_summary/analysis_summary.tsx index b3bf4ea4daf83..b80db3b27fa7e 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/analysis_summary/analysis_summary.js +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/analysis_summary/analysis_summary.tsx @@ -5,11 +5,12 @@ */ import { FormattedMessage } from '@kbn/i18n/react'; -import React from 'react'; +import React, { FC } from 'react'; import { EuiTitle, EuiSpacer, EuiDescriptionList } from '@elastic/eui'; +import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer'; -export function AnalysisSummary({ results }) { +export const AnalysisSummary: FC<{ results: FindFileStructureResponse }> = ({ results }) => { const items = createDisplayItems(results); return ( @@ -28,10 +29,10 @@ export function AnalysisSummary({ results }) { ); -} +}; -function createDisplayItems(results) { - const items = [ +function createDisplayItems(results: FindFileStructureResponse) { + const items: Array<{ title: any; description: string | number }> = [ { title: ( = ({ tooltipContent }) => { return ( ); -} +}; diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/experimental_badge/index.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/experimental_badge/index.ts similarity index 100% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/experimental_badge/index.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/experimental_badge/index.ts diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/explanation_flyout/explanation_flyout.tsx b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/explanation_flyout/explanation_flyout.tsx new file mode 100644 index 0000000000000..8e44a296126b6 --- /dev/null +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/explanation_flyout/explanation_flyout.tsx @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FC } from 'react'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { + EuiFlyout, + EuiFlyoutFooter, + EuiFlexGroup, + EuiFlexItem, + EuiFlyoutHeader, + EuiButtonEmpty, + EuiTitle, + EuiFlyoutBody, + EuiSpacer, + EuiText, + EuiSubSteps, +} from '@elastic/eui'; +import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer'; + +interface Props { + results: FindFileStructureResponse; + closeFlyout(): void; +} +export const ExplanationFlyout: FC = ({ results, closeFlyout }) => { + const explanation = results.explanation!; + return ( + + + +

+ +

+ + + + + + + + + + + + + + + + ); +}; + +const Content: FC<{ explanation: string[] }> = ({ explanation }) => ( + <> + + + + + +
    + {explanation.map((e, i) => ( +
  • + {e} + +
    • + ))} +
+ + + +); diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_progress/index.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/explanation_flyout/index.ts similarity index 78% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_progress/index.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/explanation_flyout/index.ts index f75d869f4667c..e288050403887 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_progress/index.js +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/explanation_flyout/index.ts @@ -4,4 +4,4 @@ * you may not use this file except in compliance with the Elastic License. */ -export { ImportProgress, IMPORT_STATUS } from './import_progress'; +export { ExplanationFlyout } from './explanation_flyout'; diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_contents/file_contents.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_contents/file_contents.tsx similarity index 83% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_contents/file_contents.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_contents/file_contents.tsx index ed5ab57a2588d..6564b9a1f4d83 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_contents/file_contents.js +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_contents/file_contents.tsx @@ -5,13 +5,19 @@ */ import { FormattedMessage } from '@kbn/i18n/react'; -import React from 'react'; +import React, { FC } from 'react'; import { EuiTitle, EuiSpacer } from '@elastic/eui'; import { MLJobEditor, ML_EDITOR_MODE } from '../../../../jobs/jobs_list/components/ml_job_editor'; -export function FileContents({ data, format, numberOfLines }) { +interface Props { + data: string; + format: string; + numberOfLines: number; +} + +export const FileContents: FC = ({ data, format, numberOfLines }) => { let mode = ML_EDITOR_MODE.TEXT; if (format === ML_EDITOR_MODE.JSON) { mode = ML_EDITOR_MODE.JSON; @@ -35,7 +41,7 @@ export function FileContents({ data, format, numberOfLines }) { id="xpack.ml.fileDatavisualizer.fileContents.firstLinesDescription" defaultMessage="First {numberOfLines, plural, zero {# line} one {# line} other {# lines}}" values={{ - numberOfLines: numberOfLines, + numberOfLines, }} /> @@ -51,9 +57,9 @@ export function FileContents({ data, format, numberOfLines }) { /> ); -} +}; -function limitByNumberOfLines(data, numberOfLines) { +function limitByNumberOfLines(data: string, numberOfLines: number) { return data .split('\n') .slice(0, numberOfLines) diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_contents/index.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_contents/index.ts similarity index 100% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_contents/index.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_contents/index.ts diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_datavisualizer_view/file_datavisualizer_view.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_datavisualizer_view/file_datavisualizer_view.js index 12e5a14b51871..02f14a9e4553e 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_datavisualizer_view/file_datavisualizer_view.js +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_datavisualizer_view/file_datavisualizer_view.js @@ -17,9 +17,9 @@ import { BottomBar } from '../bottom_bar'; import { ResultsView } from '../results_view'; import { FileCouldNotBeRead, FileTooLarge } from './file_error_callouts'; import { EditFlyout } from '../edit_flyout'; +import { ExplanationFlyout } from '../explanation_flyout'; import { ImportView } from '../import_view'; import { MAX_BYTES } from '../../../../../../common/constants/file_datavisualizer'; -import { isErrorResponse } from '../../../../../../common/types/errors'; import { readFile, createUrlOverrides, @@ -42,12 +42,14 @@ export class FileDataVisualizerView extends Component { fileSize: 0, fileTooLarge: false, fileCouldNotBeRead: false, - serverErrorMessage: '', + serverError: null, loading: false, loaded: false, results: undefined, + explanation: undefined, mode: MODE.READ, isEditFlyoutVisible: false, + isExplanationFlyoutVisible: false, bottomBarVisible: false, hasPermissionToImport: false, }; @@ -78,8 +80,9 @@ export class FileDataVisualizerView extends Component { fileSize: 0, fileTooLarge: false, fileCouldNotBeRead: false, - serverErrorMessage: '', + serverError: null, results: undefined, + explanation: undefined, }, () => { if (files.length) { @@ -128,7 +131,7 @@ export class FileDataVisualizerView extends Component { console.log('overrides', overrides); const { analyzeFile } = ml.fileDatavisualizer; const resp = await analyzeFile(lessData, overrides); - const serverSettings = processResults(resp.results); + const serverSettings = processResults(resp); const serverOverrides = resp.overrides; this.previousOverrides = this.overrides; @@ -172,6 +175,7 @@ export class FileDataVisualizerView extends Component { this.setState({ results: resp.results, + explanation: resp.explanation, loaded: true, loading: false, fileCouldNotBeRead: isRetry, @@ -179,19 +183,13 @@ export class FileDataVisualizerView extends Component { } catch (error) { console.error(error); - let serverErrorMsg; - if (isErrorResponse(error) === true) { - serverErrorMsg = `${error.body.error}: ${error.body.message}`; - } else { - serverErrorMsg = JSON.stringify(error, null, 2); - } - this.setState({ results: undefined, + explanation: undefined, loaded: false, loading: false, fileCouldNotBeRead: true, - serverErrorMessage: serverErrorMsg, + serverError: error, }); // as long as the previous overrides are different to the current overrides, @@ -216,6 +214,16 @@ export class FileDataVisualizerView extends Component { this.hideBottomBar(); }; + closeExplanationFlyout = () => { + this.setState({ isExplanationFlyoutVisible: false }); + this.showBottomBar(); + }; + + showExplanationFlyout = () => { + this.setState({ isExplanationFlyoutVisible: true }); + this.hideBottomBar(); + }; + showBottomBar = () => { this.setState({ bottomBarVisible: true }); }; @@ -252,14 +260,16 @@ export class FileDataVisualizerView extends Component { loading, loaded, results, + explanation, fileContents, fileName, fileSize, fileTooLarge, fileCouldNotBeRead, - serverErrorMessage, + serverError, mode, isEditFlyoutVisible, + isExplanationFlyoutVisible, bottomBarVisible, hasPermissionToImport, } = this.state; @@ -281,7 +291,7 @@ export class FileDataVisualizerView extends Component { {fileCouldNotBeRead && loading === false && ( - + )} @@ -289,9 +299,12 @@ export class FileDataVisualizerView extends Component { {loaded && ( this.showEditFlyout()} + showExplanationFlyout={() => this.showExplanationFlyout()} + disableButtons={isEditFlyoutVisible || isExplanationFlyoutVisible} /> )} + {isExplanationFlyoutVisible && ( + + )} + {bottomBarVisible && loaded && ( = ({ fileSize, maxFileSize }) => { const fileSizeFormatted = numeral(fileSize).format(FILE_SIZE_DISPLAY_FORMAT); const maxFileSizeFormatted = numeral(maxFileSize).format(FILE_SIZE_DISPLAY_FORMAT); @@ -67,9 +73,15 @@ export function FileTooLarge({ fileSize, maxFileSize }) { {errorText} ); +}; + +interface FileCouldNotBeReadProps { + error: ErrorResponse; + loaded: boolean; } -export function FileCouldNotBeRead({ error, loaded }) { +export const FileCouldNotBeRead: FC = ({ error, loaded }) => { + const message = error.body.message; return ( - {error !== undefined &&

{error}

} + {message} + {loaded && ( -

+ <> + -

+ )} ); -} +}; + +export const Explanation: FC<{ error: ErrorResponse }> = ({ error }) => { + if (!error.body.attributes?.body?.error?.suppressed?.length) { + return null; + } + const reason: string = error.body.attributes.body.error.suppressed[0].reason; + return ( + <> + + {reason.split('\n').map((m, i) => ( +
{m}
+ ))} + + ); +}; diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_errors/errors.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_errors/errors.tsx similarity index 85% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_errors/errors.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_errors/errors.tsx index 6629c0109feaf..f723ad1a752bf 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_errors/errors.js +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_errors/errors.tsx @@ -5,13 +5,24 @@ */ import { FormattedMessage } from '@kbn/i18n/react'; -import React from 'react'; +import React, { FC } from 'react'; +import { i18n } from '@kbn/i18n'; import { EuiCallOut, EuiAccordion } from '@elastic/eui'; -import { IMPORT_STATUS } from '../import_progress'; +import { IMPORT_STATUS, Statuses } from '../import_progress'; -export function ImportErrors({ errors, statuses }) { +interface ImportError { + msg: string; + more?: string; +} + +interface Props { + errors: any[]; + statuses: Statuses; +} + +export const ImportErrors: FC = ({ errors, statuses }) => { return ( {errors.map((e, i) => ( @@ -19,9 +30,9 @@ export function ImportErrors({ errors, statuses }) { ))} ); -} +}; -function title(statuses) { +function title(statuses: Statuses) { switch (IMPORT_STATUS.FAILED) { case statuses.readStatus: return ( @@ -82,7 +93,7 @@ function title(statuses) { } } -function ImportError(error, key) { +function ImportError(error: any, key: number) { const errorObj = toString(error); return ( @@ -106,7 +117,7 @@ function ImportError(error, key) { ); } -function toString(error) { +function toString(error: any): ImportError { if (typeof error === 'string') { return { msg: error }; } @@ -118,7 +129,7 @@ function toString(error) { if (typeof error.error === 'object') { if (error.error.msg !== undefined) { // this will catch a bulk ingest failure - const errorObj = { msg: error.error.msg }; + const errorObj: ImportError = { msg: error.error.msg }; if (error.error.body !== undefined) { errorObj.more = error.error.response; } @@ -139,11 +150,8 @@ function toString(error) { } return { - msg: ( - - ), + msg: i18n.translate('xpack.ml.fileDatavisualizer.importErrors.unknownErrorMessage', { + defaultMessage: 'Unknown error', + }), }; } diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_errors/index.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_errors/index.ts similarity index 100% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_errors/index.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_errors/index.ts diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_progress/import_progress.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_progress/import_progress.tsx similarity index 92% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_progress/import_progress.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_progress/import_progress.tsx index 272ec2979ad2f..533fec4ac50c8 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_progress/import_progress.js +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_progress/import_progress.tsx @@ -6,17 +6,31 @@ import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; -import React from 'react'; +import React, { FC } from 'react'; import { EuiStepsHorizontal, EuiProgress, EuiSpacer } from '@elastic/eui'; -export const IMPORT_STATUS = { - INCOMPLETE: 'incomplete', - COMPLETE: 'complete', - FAILED: 'danger', -}; +export enum IMPORT_STATUS { + INCOMPLETE = 'incomplete', + COMPLETE = 'complete', + FAILED = 'danger', +} + +export interface Statuses { + reading: boolean; + readStatus: IMPORT_STATUS; + parseJSONStatus: IMPORT_STATUS; + indexCreatedStatus: IMPORT_STATUS; + ingestPipelineCreatedStatus: IMPORT_STATUS; + indexPatternCreatedStatus: IMPORT_STATUS; + uploadProgress: number; + uploadStatus: IMPORT_STATUS; + createIndexPattern: boolean; + createPipeline: boolean; + permissionCheckStatus: IMPORT_STATUS; +} -export function ImportProgress({ statuses }) { +export const ImportProgress: FC<{ statuses: Statuses }> = ({ statuses }) => { const { reading, readStatus, @@ -271,9 +285,9 @@ export function ImportProgress({ statuses }) { )} ); -} +}; -function UploadFunctionProgress({ progress }) { +const UploadFunctionProgress: FC<{ progress: number }> = ({ progress }) => { return (

@@ -290,4 +304,4 @@ function UploadFunctionProgress({ progress }) { )} ); -} +}; diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_progress/index.ts b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_progress/index.ts new file mode 100644 index 0000000000000..9b0c6ca2264cb --- /dev/null +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_progress/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { ImportProgress, IMPORT_STATUS, Statuses } from './import_progress'; diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/advanced.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/advanced.tsx similarity index 85% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/advanced.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/advanced.tsx index 14cbe67662ed6..a79a7d36f3294 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/advanced.js +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/advanced.tsx @@ -6,7 +6,7 @@ import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; -import React from 'react'; +import React, { FC } from 'react'; import { EuiFieldText, @@ -20,7 +20,25 @@ import { import { MLJobEditor, ML_EDITOR_MODE } from '../../../../jobs/jobs_list/components/ml_job_editor'; const EDITOR_HEIGHT = '300px'; -export function AdvancedSettings({ +interface Props { + index: string; + indexPattern: string; + initialized: boolean; + onIndexChange(): void; + createIndexPattern: boolean; + onCreateIndexPatternChange(): void; + onIndexPatternChange(): void; + indexSettingsString: string; + mappingsString: string; + pipelineString: string; + onIndexSettingsStringChange(): void; + onMappingsStringChange(): void; + onPipelineStringChange(): void; + indexNameError: string; + indexPatternNameError: string; +} + +export const AdvancedSettings: FC = ({ index, indexPattern, initialized, @@ -36,7 +54,7 @@ export function AdvancedSettings({ onPipelineStringChange, indexNameError, indexPatternNameError, -}) { +}) => { return ( } - disabled={createIndexPattern === false || initialized === true} isInvalid={indexPatternNameError !== ''} error={[indexPatternNameError]} > @@ -133,9 +150,15 @@ export function AdvancedSettings({ ); +}; + +interface JsonEditorProps { + initialized: boolean; + data: string; + onChange(): void; } -function IndexSettings({ initialized, data, onChange }) { +const IndexSettings: FC = ({ initialized, data, onChange }) => { return ( } - disabled={initialized === true} fullWidth > ); -} +}; -function Mappings({ initialized, data, onChange }) { +const Mappings: FC = ({ initialized, data, onChange }) => { return ( } - disabled={initialized === true} fullWidth > ); -} +}; -function IngestPipeline({ initialized, data, onChange }) { +const IngestPipeline: FC = ({ initialized, data, onChange }) => { return ( } - disabled={initialized === true} fullWidth > ); -} +}; diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/import_settings.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/import_settings.tsx similarity index 82% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/import_settings.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/import_settings.tsx index ba637c472333d..02cf647caaf16 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/import_settings.js +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/import_settings.tsx @@ -5,14 +5,32 @@ */ import { i18n } from '@kbn/i18n'; -import React from 'react'; +import React, { FC } from 'react'; import { EuiTabbedContent, EuiSpacer } from '@elastic/eui'; import { SimpleSettings } from './simple'; import { AdvancedSettings } from './advanced'; -export const ImportSettings = ({ +interface Props { + index: string; + indexPattern: string; + initialized: boolean; + onIndexChange(): void; + createIndexPattern: boolean; + onCreateIndexPatternChange(): void; + onIndexPatternChange(): void; + indexSettingsString: string; + mappingsString: string; + pipelineString: string; + onIndexSettingsStringChange(): void; + onMappingsStringChange(): void; + onPipelineStringChange(): void; + indexNameError: string; + indexPatternNameError: string; +} + +export const ImportSettings: FC = ({ index, indexPattern, initialized, diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/index.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/index.ts similarity index 100% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/index.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/index.ts diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/simple.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/simple.tsx similarity index 88% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/simple.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/simple.tsx index 271b9493aa1f3..1e716824729e3 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/simple.js +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/simple.tsx @@ -6,11 +6,20 @@ import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; -import React from 'react'; +import React, { FC } from 'react'; import { EuiFieldText, EuiFormRow, EuiCheckbox, EuiSpacer } from '@elastic/eui'; -export const SimpleSettings = ({ +interface Props { + index: string; + initialized: boolean; + onIndexChange(): void; + createIndexPattern: boolean; + onCreateIndexPatternChange(): void; + indexNameError: string; +} + +export const SimpleSettings: FC = ({ index, initialized, onIndexChange, diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_summary/import_summary.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_summary/import_summary.tsx similarity index 86% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_summary/import_summary.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_summary/import_summary.tsx index 0e67807a39fd9..6ee17d401bd70 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_summary/import_summary.js +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_summary/import_summary.tsx @@ -5,11 +5,29 @@ */ import { FormattedMessage } from '@kbn/i18n/react'; -import React from 'react'; +import React, { FC } from 'react'; import { EuiSpacer, EuiDescriptionList, EuiCallOut, EuiAccordion } from '@elastic/eui'; -export function ImportSummary({ +interface Props { + index: string; + indexPattern: string; + ingestPipelineId: string; + docCount: number; + importFailures: DocFailure[]; + createIndexPattern: boolean; + createPipeline: boolean; +} + +interface DocFailure { + item: number; + reason: string; + doc: { + message: string; + }; +} + +export const ImportSummary: FC = ({ index, indexPattern, ingestPipelineId, @@ -17,7 +35,7 @@ export function ImportSummary({ importFailures, createIndexPattern, createPipeline, -}) { +}) => { const items = createDisplayItems( index, indexPattern, @@ -75,9 +93,13 @@ export function ImportSummary({ )} ); +}; + +interface FailuresProps { + failedDocs: DocFailure[]; } -function Failures({ failedDocs }) { +const Failures: FC = ({ failedDocs }) => { return ( ); -} +}; function createDisplayItems( - index, - indexPattern, - ingestPipelineId, - docCount, - importFailures, - createIndexPattern, - createPipeline + index: string, + indexPattern: string, + ingestPipelineId: string, + docCount: number, + importFailures: DocFailure[], + createIndexPattern: boolean, + createPipeline: boolean ) { const items = [ { diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_summary/index.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_summary/index.ts similarity index 100% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_summary/index.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_summary/index.ts diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/import_view.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/import_view.js index 0a58153e374df..4c9579bfd4b46 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/import_view.js +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/import_view.js @@ -623,7 +623,6 @@ async function createKibanaIndexPattern( id, }; } catch (error) { - console.error(error); return { success: false, error, diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/importer.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/importer.ts similarity index 73% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/importer.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/importer.ts index 27899a58beed2..c97f1c147c454 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/importer.js +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/importer.ts @@ -4,30 +4,53 @@ * you may not use this file except in compliance with the Elastic License. */ -import { ml } from '../../../../../services/ml_api_service'; import { chunk } from 'lodash'; import moment from 'moment'; import { i18n } from '@kbn/i18n'; +import { ml } from '../../../../../services/ml_api_service'; +import { + Doc, + ImportFailure, + ImportResponse, + Mappings, + Settings, + IngestPipeline, +} from '../../../../../../../common/types/file_datavisualizer'; const CHUNK_SIZE = 5000; const MAX_CHUNK_CHAR_COUNT = 1000000; const IMPORT_RETRIES = 5; +export interface ImportConfig { + settings: Settings; + mappings: Mappings; + pipeline: IngestPipeline; +} + +export interface ImportResults { + success: boolean; + failures?: any[]; + docCount?: number; + error?: any; +} + export class Importer { - constructor({ settings, mappings, pipeline }) { - this.settings = settings; - this.mappings = mappings; - this.pipeline = pipeline; - - this.data = []; - this.docArray = []; - this.docSizeArray = []; + private _settings: Settings; + private _mappings: Mappings; + private _pipeline: IngestPipeline; + + protected _docArray: Doc[] = []; + + constructor({ settings, mappings, pipeline }: ImportConfig) { + this._settings = settings; + this._mappings = mappings; + this._pipeline = pipeline; } - async initializeImport(index) { - const settings = this.settings; - const mappings = this.mappings; - const pipeline = this.pipeline; + async initializeImport(index: string) { + const settings = this._settings; + const mappings = this._mappings; + const pipeline = this._pipeline; updatePipelineTimezone(pipeline); // if no pipeline has been supplied, @@ -52,7 +75,12 @@ export class Importer { return createIndexResp; } - async import(id, index, pipelineId, setImportProgress) { + async import( + id: string, + index: string, + pipelineId: string, + setImportProgress: (progress: number) => void + ): Promise { if (!id || !index) { return { success: false, @@ -65,14 +93,14 @@ export class Importer { }; } - const chunks = createDocumentChunks(this.docArray); + const chunks = createDocumentChunks(this._docArray); const ingestPipeline = { id: pipelineId, }; let success = true; - const failures = []; + const failures: ImportFailure[] = []; let error; for (let i = 0; i < chunks.length; i++) { @@ -86,10 +114,13 @@ export class Importer { }; let retries = IMPORT_RETRIES; - let resp = { + let resp: ImportResponse = { success: false, failures: [], docCount: 0, + id: '', + index: '', + pipelineId: '', }; while (resp.success === false && retries > 0) { @@ -97,12 +128,14 @@ export class Importer { resp = await ml.fileDatavisualizer.import(aggs); if (retries < IMPORT_RETRIES) { + // eslint-disable-next-line no-console console.log(`Retrying import ${IMPORT_RETRIES - retries}`); } retries--; } catch (err) { - resp = { success: false, error: err }; + resp.success = false; + resp.error = err; retries = 0; } } @@ -110,6 +143,7 @@ export class Importer { if (resp.success) { setImportProgress(((i + 1) / chunks.length) * 100); } else { + // eslint-disable-next-line no-console console.error(resp); success = false; error = resp.error; @@ -120,10 +154,10 @@ export class Importer { populateFailures(resp, failures, i); } - const result = { + const result: ImportResults = { success, failures, - docCount: this.docArray.length, + docCount: this._docArray.length, }; if (success) { @@ -136,7 +170,7 @@ export class Importer { } } -function populateFailures(error, failures, chunkCount) { +function populateFailures(error: ImportResponse, failures: ImportFailure[], chunkCount: number) { if (error.failures && error.failures.length) { // update the item value to include the chunk count // e.g. item 3 in chunk 2 is actually item 20003 @@ -155,10 +189,10 @@ function populateFailures(error, failures, chunkCount) { // But it's not sending every single field that Filebeat would add, so the ingest pipeline // cannot look for a event.timezone variable in each input record. // Therefore we need to replace {{ event.timezone }} with the actual browser timezone -function updatePipelineTimezone(ingestPipeline) { +function updatePipelineTimezone(ingestPipeline: IngestPipeline) { if (ingestPipeline !== undefined && ingestPipeline.processors && ingestPipeline.processors) { const dateProcessor = ingestPipeline.processors.find( - p => p.date !== undefined && p.date.timezone === '{{ event.timezone }}' + (p: any) => p.date !== undefined && p.date.timezone === '{{ event.timezone }}' ); if (dateProcessor) { @@ -167,8 +201,8 @@ function updatePipelineTimezone(ingestPipeline) { } } -function createDocumentChunks(docArray) { - const chunks = []; +function createDocumentChunks(docArray: Doc[]) { + const chunks: Doc[][] = []; // chop docArray into 5000 doc chunks const tempChunks = chunk(docArray, CHUNK_SIZE); diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/importer_factory.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/importer_factory.ts similarity index 76% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/importer_factory.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/importer_factory.ts index 381e8ef604452..a656b19220368 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/importer_factory.js +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/importer_factory.ts @@ -6,8 +6,14 @@ import { MessageImporter } from './message_importer'; import { NdjsonImporter } from './ndjson_importer'; +import { ImportConfig } from './importer'; +import { FindFileStructureResponse } from '../../../../../../../common/types/file_datavisualizer'; -export function importerFactory(format, results, settings) { +export function importerFactory( + format: string, + results: FindFileStructureResponse, + settings: ImportConfig +) { switch (format) { // delimited and semi-structured text are both handled by splitting the // file into messages, then sending these to ES for further processing diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/index.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/index.ts similarity index 100% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/index.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/index.ts diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/message_importer.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/message_importer.ts similarity index 76% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/message_importer.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/message_importer.ts index c2d3ac69f0963..7ccc5a8d673f4 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/message_importer.js +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/message_importer.ts @@ -4,17 +4,24 @@ * you may not use this file except in compliance with the Elastic License. */ -import { Importer } from './importer'; +import { Importer, ImportConfig } from './importer'; +import { + Doc, + FindFileStructureResponse, +} from '../../../../../../../common/types/file_datavisualizer'; export class MessageImporter extends Importer { - constructor(results, settings) { + private _excludeLinesRegex: RegExp | null; + private _multilineStartRegex: RegExp | null; + + constructor(results: FindFileStructureResponse, settings: ImportConfig) { super(settings); - this.excludeLinesRegex = + this._excludeLinesRegex = results.exclude_lines_pattern === undefined ? null : new RegExp(results.exclude_lines_pattern); - this.multilineStartRegex = + this._multilineStartRegex = results.multiline_start_pattern === undefined ? null : new RegExp(results.multiline_start_pattern); @@ -26,9 +33,9 @@ export class MessageImporter extends Importer { // multiline_start_pattern regex // if it does, it is a legitimate end of line and can be pushed into the list, // if not, it must be a newline char inside a field value, so keep looking. - read(text) { + read(text: string) { try { - const data = []; + const data: Doc[] = []; let message = ''; let line = ''; @@ -57,14 +64,12 @@ export class MessageImporter extends Importer { data.shift(); } - this.data = data; - this.docArray = this.data; + this._docArray = data; return { success: true, }; } catch (error) { - console.error(error); return { success: false, error, @@ -72,9 +77,9 @@ export class MessageImporter extends Importer { } } - processLine(data, message, line) { - if (this.excludeLinesRegex === null || line.match(this.excludeLinesRegex) === null) { - if (this.multilineStartRegex === null || line.match(this.multilineStartRegex) !== null) { + processLine(data: Doc[], message: string, line: string) { + if (this._excludeLinesRegex === null || line.match(this._excludeLinesRegex) === null) { + if (this._multilineStartRegex === null || line.match(this._multilineStartRegex) !== null) { this.addMessage(data, message); message = ''; } else if (data.length === 0) { @@ -90,7 +95,7 @@ export class MessageImporter extends Importer { return message; } - addMessage(data, message) { + addMessage(data: Doc[], message: string) { // if the message ended \r\n (Windows line endings) // then omit the \r as well as the \n for consistency message = message.replace(/\r$/, ''); diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/ndjson_importer.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/ndjson_importer.ts similarity index 71% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/ndjson_importer.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/ndjson_importer.ts index 887bf1a41200a..7f5f37abc5246 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/ndjson_importer.js +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/ndjson_importer.ts @@ -4,18 +4,19 @@ * you may not use this file except in compliance with the Elastic License. */ -import { Importer } from './importer'; +import { Importer, ImportConfig } from './importer'; +import { FindFileStructureResponse } from '../../../../../../../common/types/file_datavisualizer'; export class NdjsonImporter extends Importer { - constructor(results, settings) { + constructor(results: FindFileStructureResponse, settings: ImportConfig) { super(settings); } - read(json) { + read(json: string) { try { const splitJson = json.split(/}\s*\n/); - const ndjson = []; + const ndjson: any[] = []; for (let i = 0; i < splitJson.length; i++) { if (splitJson[i] !== '') { // note the extra } at the end of the line, adding back @@ -24,7 +25,7 @@ export class NdjsonImporter extends Importer { } } - this.docArray = ndjson; + this._docArray = ndjson; return { success: true, diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/results_view/index.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/results_view/index.ts similarity index 100% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/results_view/index.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/results_view/index.ts diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/results_view/results_view.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/results_view/results_view.tsx similarity index 58% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/results_view/results_view.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/results_view/results_view.tsx index 96116e5cefa01..f9de03c119d28 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/results_view/results_view.js +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/results_view/results_view.tsx @@ -7,10 +7,10 @@ import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; -import React from 'react'; - +import React, { FC } from 'react'; import { EuiButton, + EuiButtonEmpty, EuiPage, EuiPageBody, EuiPageContentHeader, @@ -18,13 +18,33 @@ import { EuiTabbedContent, EuiSpacer, EuiTitle, + EuiFlexGroup, + EuiFlexItem, } from '@elastic/eui'; +import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer'; import { FileContents } from '../file_contents'; import { AnalysisSummary } from '../analysis_summary'; +// @ts-ignore import { FieldsStats } from '../fields_stats'; -export const ResultsView = ({ data, fileName, results, showEditFlyout }) => { +interface Props { + data: string; + fileName: string; + results: FindFileStructureResponse; + showEditFlyout(): void; + showExplanationFlyout(): void; + disableButtons: boolean; +} + +export const ResultsView: FC = ({ + data, + fileName, + results, + showEditFlyout, + showExplanationFlyout, + disableButtons, +}) => { const tabs = [ { id: 'file-stats', @@ -60,12 +80,24 @@ export const ResultsView = ({ data, fileName, results, showEditFlyout }) => { - showEditFlyout()}> - - + + + showEditFlyout()} disabled={disableButtons}> + + + + + showExplanationFlyout()} disabled={disableButtons}> + + + + diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/index.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/index.ts similarity index 100% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/index.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/index.ts diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/overrides.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/overrides.js deleted file mode 100644 index f8a90c87b9dc8..0000000000000 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/overrides.js +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -export const DEFAULT_LINES_TO_SAMPLE = 1000; - -export const overrideDefaults = { - timestampFormat: undefined, - timestampField: undefined, - format: undefined, - delimiter: undefined, - quote: undefined, - hasHeaderRow: undefined, - charset: undefined, - columnNames: undefined, - shouldTrimFields: undefined, - grokPattern: undefined, - linesToSample: undefined, -}; diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/utils.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/utils.ts similarity index 79% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/utils.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/utils.ts index 39cd25ba87d8c..5048065ae60fa 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/utils.js +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/utils.ts @@ -4,11 +4,27 @@ * you may not use this file except in compliance with the Elastic License. */ -import { overrideDefaults, DEFAULT_LINES_TO_SAMPLE } from './overrides'; import { isEqual } from 'lodash'; import { ml } from '../../../../services/ml_api_service'; - -export function readFile(file) { +import { AnalysisResult, InputOverrides } from '../../../../../../common/types/file_datavisualizer'; + +const DEFAULT_LINES_TO_SAMPLE = 1000; + +const overrideDefaults = { + timestampFormat: undefined, + timestampField: undefined, + format: undefined, + delimiter: undefined, + quote: undefined, + hasHeaderRow: undefined, + charset: undefined, + columnNames: undefined, + shouldTrimFields: undefined, + grokPattern: undefined, + linesToSample: undefined, +}; + +export function readFile(file: File) { return new Promise((resolve, reject) => { if (file && file.size) { const reader = new FileReader(); @@ -23,14 +39,14 @@ export function readFile(file) { resolve({ data }); } }; - })(file); + })(); } else { reject(); } }); } -export function reduceData(data, mb) { +export function reduceData(data: string, mb: number) { // assuming ascii characters in the file where 1 char is 1 byte // TODO - change this when other non UTF-8 formats are // supported for the read data @@ -38,8 +54,8 @@ export function reduceData(data, mb) { return data.length >= size ? data.slice(0, size) : data; } -export function createUrlOverrides(overrides, originalSettings) { - const formattedOverrides = {}; +export function createUrlOverrides(overrides: InputOverrides, originalSettings: InputOverrides) { + const formattedOverrides: InputOverrides = {}; for (const o in overrideDefaults) { if (overrideDefaults.hasOwnProperty(o)) { let value = overrides[o]; @@ -93,15 +109,15 @@ export function createUrlOverrides(overrides, originalSettings) { return formattedOverrides; } -export function processResults(results) { +export function processResults({ results, overrides }: AnalysisResult) { const timestampFormat = results.java_timestamp_formats !== undefined && results.java_timestamp_formats.length ? results.java_timestamp_formats[0] : undefined; const linesToSample = - results.overrides !== undefined && results.overrides.lines_to_sample !== undefined - ? results.overrides.lines_to_sample + overrides !== undefined && overrides.lines_to_sample !== undefined + ? overrides.lines_to_sample : DEFAULT_LINES_TO_SAMPLE; return { @@ -125,8 +141,8 @@ export function processResults(results) { * @param {string} indexName * @returns {Promise} */ -export async function hasImportPermission(indexName) { - const priv = { +export async function hasImportPermission(indexName: string) { + const priv: { cluster: string[]; index?: any } = { cluster: ['cluster:monitor/nodes/info', 'cluster:admin/ingest/pipeline/put'], }; diff --git a/x-pack/plugins/ml/public/application/services/ml_api_service/datavisualizer.ts b/x-pack/plugins/ml/public/application/services/ml_api_service/datavisualizer.ts index 9b492530d303d..20332546d9cde 100644 --- a/x-pack/plugins/ml/public/application/services/ml_api_service/datavisualizer.ts +++ b/x-pack/plugins/ml/public/application/services/ml_api_service/datavisualizer.ts @@ -7,6 +7,7 @@ import { http } from '../http_service'; import { basePath } from './index'; +import { ImportResponse } from '../../../../common/types/file_datavisualizer'; export const fileDatavisualizer = { analyzeFile(file: string, params: Record = {}) { @@ -27,7 +28,7 @@ export const fileDatavisualizer = { mappings, ingestPipeline, }: { - id: string; + id: string | undefined; index: string; data: any; settings: any; @@ -43,7 +44,7 @@ export const fileDatavisualizer = { ingestPipeline, }); - return http({ + return http({ path: `${basePath()}/file_data_visualizer/import`, method: 'POST', query, diff --git a/x-pack/plugins/ml/server/client/elasticsearch_ml.ts b/x-pack/plugins/ml/server/client/elasticsearch_ml.ts index caedaed92e5b1..d5c7882a30d20 100644 --- a/x-pack/plugins/ml/server/client/elasticsearch_ml.ts +++ b/x-pack/plugins/ml/server/client/elasticsearch_ml.ts @@ -740,7 +740,7 @@ export const elasticsearchJsPlugin = (Client: any, config: any, components: any) urls: [ { fmt: - '/_ml/find_file_structure?&charset=<%=charset%>&format=<%=format%>&has_header_row=<%=has_header_row%>&column_names=<%=column_names%>&delimiter=<%=delimiter%>"e=<%=quote%>&should_trim_fields=<%=should_trim_fields%>&grok_pattern=<%=grok_pattern%>×tamp_field=<%=timestamp_field%>×tamp_format=<%=timestamp_format%>&lines_to_sample=<%=lines_to_sample%>', + '/_ml/find_file_structure?&explain=true&charset=<%=charset%>&format=<%=format%>&has_header_row=<%=has_header_row%>&column_names=<%=column_names%>&delimiter=<%=delimiter%>"e=<%=quote%>&should_trim_fields=<%=should_trim_fields%>&grok_pattern=<%=grok_pattern%>×tamp_field=<%=timestamp_field%>×tamp_format=<%=timestamp_format%>&lines_to_sample=<%=lines_to_sample%>', req: { charset: { type: 'string', @@ -778,7 +778,7 @@ export const elasticsearchJsPlugin = (Client: any, config: any, components: any) }, }, { - fmt: '/_ml/find_file_structure', + fmt: '/_ml/find_file_structure?&explain=true', }, ], needBody: true, diff --git a/x-pack/plugins/ml/server/client/error_wrapper.ts b/x-pack/plugins/ml/server/client/error_wrapper.ts index 7f69173295482..de53e4d4345a9 100644 --- a/x-pack/plugins/ml/server/client/error_wrapper.ts +++ b/x-pack/plugins/ml/server/client/error_wrapper.ts @@ -9,9 +9,13 @@ import { ResponseError, CustomHttpResponseOptions } from 'kibana/server'; export function wrapError(error: any): CustomHttpResponseOptions { const boom = isBoom(error) ? error : boomify(error, { statusCode: error.status }); + const statusCode = boom.output.statusCode; return { - body: boom, + body: { + message: boom, + ...(statusCode !== 500 && error.body ? { attributes: { body: error.body } } : {}), + }, headers: boom.output.headers, - statusCode: boom.output.statusCode, + statusCode, }; } diff --git a/x-pack/plugins/ml/server/models/file_data_visualizer/file_data_visualizer.ts b/x-pack/plugins/ml/server/models/file_data_visualizer/file_data_visualizer.ts index 9af755c6918fb..d53378b886a99 100644 --- a/x-pack/plugins/ml/server/models/file_data_visualizer/file_data_visualizer.ts +++ b/x-pack/plugins/ml/server/models/file_data_visualizer/file_data_visualizer.ts @@ -4,40 +4,21 @@ * you may not use this file except in compliance with the Elastic License. */ -import Boom from 'boom'; import { APICaller } from 'kibana/server'; -import { FindFileStructureResponse } from '../../../common/types/file_datavisualizer'; +import { + AnalysisResult, + FormattedOverrides, + InputOverrides, +} from '../../../common/types/file_datavisualizer'; export type InputData = any[]; -export interface InputOverrides { - [key: string]: string; -} - -export type FormattedOverrides = InputOverrides & { - column_names: string[]; - has_header_row: boolean; - should_trim_fields: boolean; -}; - -export interface AnalysisResult { - results: FindFileStructureResponse; - overrides?: FormattedOverrides; -} - export function fileDataVisualizerProvider(callAsCurrentUser: APICaller) { async function analyzeFile(data: any, overrides: any): Promise { - let results = []; - - try { - results = await callAsCurrentUser('ml.fileStructure', { - body: data, - ...overrides, - }); - } catch (error) { - const err = error.message !== undefined ? error.message : error; - throw Boom.badRequest(err); - } + const results = await callAsCurrentUser('ml.fileStructure', { + body: data, + ...overrides, + }); const { hasOverrides, reducedOverrides } = formatOverrides(overrides); diff --git a/x-pack/plugins/ml/server/models/file_data_visualizer/import_data.ts b/x-pack/plugins/ml/server/models/file_data_visualizer/import_data.ts index ab8c702cbb12a..9d7009955124f 100644 --- a/x-pack/plugins/ml/server/models/file_data_visualizer/import_data.ts +++ b/x-pack/plugins/ml/server/models/file_data_visualizer/import_data.ts @@ -6,39 +6,24 @@ import { APICaller } from 'kibana/server'; import { INDEX_META_DATA_CREATED_BY } from '../../../common/constants/file_datavisualizer'; +import { + ImportResponse, + ImportFailure, + Settings, + Mappings, + IngestPipelineWrapper, +} from '../../../common/types/file_datavisualizer'; import { InputData } from './file_data_visualizer'; -export interface Settings { - pipeline?: string; - index: string; - body: any[]; - [key: string]: any; -} - -export interface Mappings { - [key: string]: any; -} - -export interface InjectPipeline { - id: string; - pipeline: any; -} - -interface Failure { - item: number; - reason: string; - doc: any; -} - export function importDataProvider(callAsCurrentUser: APICaller) { async function importData( id: string, index: string, settings: Settings, mappings: Mappings, - ingestPipeline: InjectPipeline, + ingestPipeline: IngestPipelineWrapper, data: InputData - ) { + ): Promise { let createdIndex; let createdPipelineId; const docCount = data.length; @@ -66,7 +51,7 @@ export function importDataProvider(callAsCurrentUser: APICaller) { createdPipelineId = pipelineId; } - let failures: Failure[] = []; + let failures: ImportFailure[] = []; if (data.length) { const resp = await indexData(index, createdPipelineId, data); if (resp.success === false) { @@ -144,7 +129,7 @@ export function importDataProvider(callAsCurrentUser: APICaller) { }; } } catch (error) { - let failures: Failure[] = []; + let failures: ImportFailure[] = []; let ingestError = false; if (error.errors !== undefined && Array.isArray(error.items)) { // an expected error where some or all of the bulk request @@ -169,7 +154,7 @@ export function importDataProvider(callAsCurrentUser: APICaller) { return await callAsCurrentUser('ingest.putPipeline', { id, body: pipeline }); } - function getFailures(items: any[], data: InputData): Failure[] { + function getFailures(items: any[], data: InputData): ImportFailure[] { const failures = []; for (let i = 0; i < items.length; i++) { const item = items[i]; diff --git a/x-pack/plugins/ml/server/models/file_data_visualizer/index.ts b/x-pack/plugins/ml/server/models/file_data_visualizer/index.ts index 94529dc111696..f8a27fdcd7e1a 100644 --- a/x-pack/plugins/ml/server/models/file_data_visualizer/index.ts +++ b/x-pack/plugins/ml/server/models/file_data_visualizer/index.ts @@ -4,11 +4,6 @@ * you may not use this file except in compliance with the Elastic License. */ -export { - fileDataVisualizerProvider, - InputOverrides, - InputData, - AnalysisResult, -} from './file_data_visualizer'; +export { fileDataVisualizerProvider, InputData } from './file_data_visualizer'; -export { importDataProvider, Settings, InjectPipeline, Mappings } from './import_data'; +export { importDataProvider } from './import_data'; diff --git a/x-pack/plugins/ml/server/routes/file_data_visualizer.ts b/x-pack/plugins/ml/server/routes/file_data_visualizer.ts index fcfd6e121c9f1..b915d13aa9720 100644 --- a/x-pack/plugins/ml/server/routes/file_data_visualizer.ts +++ b/x-pack/plugins/ml/server/routes/file_data_visualizer.ts @@ -7,15 +7,17 @@ import { schema } from '@kbn/config-schema'; import { RequestHandlerContext } from 'kibana/server'; import { MAX_BYTES } from '../../common/constants/file_datavisualizer'; -import { wrapError } from '../client/error_wrapper'; import { InputOverrides, + Settings, + IngestPipelineWrapper, + Mappings, +} from '../../common/types/file_datavisualizer'; +import { wrapError } from '../client/error_wrapper'; +import { InputData, fileDataVisualizerProvider, importDataProvider, - Settings, - InjectPipeline, - Mappings, } from '../models/file_data_visualizer'; import { RouteInitialization } from '../types'; @@ -32,7 +34,7 @@ function importData( index: string, settings: Settings, mappings: Mappings, - ingestPipeline: InjectPipeline, + ingestPipeline: IngestPipelineWrapper, data: InputData ) { const { importData: importDataFunc } = importDataProvider(context.ml!.mlClient.callAsCurrentUser); diff --git a/x-pack/plugins/monitoring/common/constants.ts b/x-pack/plugins/monitoring/common/constants.ts index 9a4030f3eb214..3a4c7b71dcd03 100644 --- a/x-pack/plugins/monitoring/common/constants.ts +++ b/x-pack/plugins/monitoring/common/constants.ts @@ -239,11 +239,15 @@ export const ALERT_TYPE_PREFIX = 'monitoring_'; * This is the alert type id for the license expiration alert */ export const ALERT_TYPE_LICENSE_EXPIRATION = `${ALERT_TYPE_PREFIX}alert_type_license_expiration`; +/** + * This is the alert type id for the cluster state alert + */ +export const ALERT_TYPE_CLUSTER_STATE = `${ALERT_TYPE_PREFIX}alert_type_cluster_state`; /** * A listing of all alert types */ -export const ALERT_TYPES = [ALERT_TYPE_LICENSE_EXPIRATION]; +export const ALERT_TYPES = [ALERT_TYPE_LICENSE_EXPIRATION, ALERT_TYPE_CLUSTER_STATE]; /** * Matches the id for the built-in in email action type @@ -254,7 +258,7 @@ export const ALERT_ACTION_TYPE_EMAIL = '.email'; /** * The number of alerts that have been migrated */ -export const NUMBER_OF_MIGRATED_ALERTS = 1; +export const NUMBER_OF_MIGRATED_ALERTS = 2; /** * The advanced settings config name for the email address diff --git a/x-pack/plugins/monitoring/server/alerts/cluster_state.test.ts b/x-pack/plugins/monitoring/server/alerts/cluster_state.test.ts new file mode 100644 index 0000000000000..6a9ca88437347 --- /dev/null +++ b/x-pack/plugins/monitoring/server/alerts/cluster_state.test.ts @@ -0,0 +1,186 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { Logger } from 'src/core/server'; +import { savedObjectsClientMock } from 'src/core/server/mocks'; +import { getClusterState } from './cluster_state'; +import { AlertServices } from '../../../alerting/server'; +import { ALERT_TYPE_CLUSTER_STATE } from '../../common/constants'; +import { AlertCommonParams, AlertCommonState, AlertClusterStatePerClusterState } from './types'; +import { getPreparedAlert } from '../lib/alerts/get_prepared_alert'; +import { executeActions } from '../lib/alerts/cluster_state.lib'; +import { AlertClusterStateState } from './enums'; + +jest.mock('../lib/alerts/cluster_state.lib', () => ({ + executeActions: jest.fn(), + getUiMessage: jest.fn(), +})); + +jest.mock('../lib/alerts/get_prepared_alert', () => ({ + getPreparedAlert: jest.fn(() => { + return { + emailAddress: 'foo@foo.com', + }; + }), +})); + +interface MockServices { + callCluster: jest.Mock; + alertInstanceFactory: jest.Mock; + savedObjectsClient: jest.Mock; +} + +describe('getClusterState', () => { + const services: MockServices | AlertServices = { + callCluster: jest.fn(), + alertInstanceFactory: jest.fn(), + savedObjectsClient: savedObjectsClientMock.create(), + }; + + const params: AlertCommonParams = { + dateFormat: 'YYYY', + timezone: 'UTC', + }; + + const emailAddress = 'foo@foo.com'; + const clusterUuid = 'kdksdfj434'; + const clusterName = 'monitoring_test'; + const cluster = { clusterUuid, clusterName }; + + async function setupAlert( + previousState: AlertClusterStateState, + newState: AlertClusterStateState + ): Promise { + const logger: Logger = { + warn: jest.fn(), + log: jest.fn(), + debug: jest.fn(), + trace: jest.fn(), + error: jest.fn(), + fatal: jest.fn(), + info: jest.fn(), + get: jest.fn(), + }; + const getLogger = (): Logger => logger; + const ccrEnabled = false; + (getPreparedAlert as jest.Mock).mockImplementation(() => ({ + emailAddress, + data: [ + { + state: newState, + clusterUuid, + }, + ], + clusters: [cluster], + })); + + const alert = getClusterState(null as any, null as any, getLogger, ccrEnabled); + const state: AlertCommonState = { + [clusterUuid]: { + state: previousState, + ui: { + isFiring: false, + severity: 0, + message: null, + resolvedMS: 0, + lastCheckedMS: 0, + triggeredMS: 0, + }, + } as AlertClusterStatePerClusterState, + }; + + return (await alert.executor({ services, params, state } as any)) as AlertCommonState; + } + + afterEach(() => { + (executeActions as jest.Mock).mockClear(); + }); + + it('should configure the alert properly', () => { + const alert = getClusterState(null as any, null as any, jest.fn(), false); + expect(alert.id).toBe(ALERT_TYPE_CLUSTER_STATE); + expect(alert.actionGroups).toEqual([{ id: 'default', name: 'Default' }]); + }); + + it('should alert if green -> yellow', async () => { + const result = await setupAlert(AlertClusterStateState.Green, AlertClusterStateState.Yellow); + expect(executeActions).toHaveBeenCalledWith( + undefined, + cluster, + AlertClusterStateState.Yellow, + emailAddress + ); + const clusterResult = result[clusterUuid] as AlertClusterStatePerClusterState; + expect(clusterResult.state).toBe(AlertClusterStateState.Yellow); + expect(clusterResult.ui.isFiring).toBe(true); + expect(clusterResult.ui.resolvedMS).toBe(0); + }); + + it('should alert if yellow -> green', async () => { + const result = await setupAlert(AlertClusterStateState.Yellow, AlertClusterStateState.Green); + expect(executeActions).toHaveBeenCalledWith( + undefined, + cluster, + AlertClusterStateState.Green, + emailAddress, + true + ); + const clusterResult = result[clusterUuid] as AlertClusterStatePerClusterState; + expect(clusterResult.state).toBe(AlertClusterStateState.Green); + expect(clusterResult.ui.resolvedMS).toBeGreaterThan(0); + }); + + it('should alert if green -> red', async () => { + const result = await setupAlert(AlertClusterStateState.Green, AlertClusterStateState.Red); + expect(executeActions).toHaveBeenCalledWith( + undefined, + cluster, + AlertClusterStateState.Red, + emailAddress + ); + const clusterResult = result[clusterUuid] as AlertClusterStatePerClusterState; + expect(clusterResult.state).toBe(AlertClusterStateState.Red); + expect(clusterResult.ui.isFiring).toBe(true); + expect(clusterResult.ui.resolvedMS).toBe(0); + }); + + it('should alert if red -> green', async () => { + const result = await setupAlert(AlertClusterStateState.Red, AlertClusterStateState.Green); + expect(executeActions).toHaveBeenCalledWith( + undefined, + cluster, + AlertClusterStateState.Green, + emailAddress, + true + ); + const clusterResult = result[clusterUuid] as AlertClusterStatePerClusterState; + expect(clusterResult.state).toBe(AlertClusterStateState.Green); + expect(clusterResult.ui.resolvedMS).toBeGreaterThan(0); + }); + + it('should not alert if red -> yellow', async () => { + const result = await setupAlert(AlertClusterStateState.Red, AlertClusterStateState.Yellow); + expect(executeActions).not.toHaveBeenCalled(); + const clusterResult = result[clusterUuid] as AlertClusterStatePerClusterState; + expect(clusterResult.state).toBe(AlertClusterStateState.Red); + expect(clusterResult.ui.resolvedMS).toBe(0); + }); + + it('should not alert if yellow -> red', async () => { + const result = await setupAlert(AlertClusterStateState.Yellow, AlertClusterStateState.Red); + expect(executeActions).not.toHaveBeenCalled(); + const clusterResult = result[clusterUuid] as AlertClusterStatePerClusterState; + expect(clusterResult.state).toBe(AlertClusterStateState.Yellow); + expect(clusterResult.ui.resolvedMS).toBe(0); + }); + + it('should not alert if green -> green', async () => { + const result = await setupAlert(AlertClusterStateState.Green, AlertClusterStateState.Green); + expect(executeActions).not.toHaveBeenCalled(); + const clusterResult = result[clusterUuid] as AlertClusterStatePerClusterState; + expect(clusterResult.state).toBe(AlertClusterStateState.Green); + expect(clusterResult.ui.resolvedMS).toBe(0); + }); +}); diff --git a/x-pack/plugins/monitoring/server/alerts/cluster_state.ts b/x-pack/plugins/monitoring/server/alerts/cluster_state.ts new file mode 100644 index 0000000000000..9a5805b8af7ce --- /dev/null +++ b/x-pack/plugins/monitoring/server/alerts/cluster_state.ts @@ -0,0 +1,134 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import moment from 'moment-timezone'; +import { i18n } from '@kbn/i18n'; +import { Logger, ICustomClusterClient, UiSettingsServiceStart } from 'src/core/server'; +import { ALERT_TYPE_CLUSTER_STATE } from '../../common/constants'; +import { AlertType } from '../../../alerting/server'; +import { executeActions, getUiMessage } from '../lib/alerts/cluster_state.lib'; +import { + AlertCommonExecutorOptions, + AlertCommonState, + AlertClusterStatePerClusterState, + AlertCommonCluster, +} from './types'; +import { AlertClusterStateState } from './enums'; +import { getPreparedAlert } from '../lib/alerts/get_prepared_alert'; +import { fetchClusterState } from '../lib/alerts/fetch_cluster_state'; + +export const getClusterState = ( + getUiSettingsService: () => Promise, + monitoringCluster: ICustomClusterClient, + getLogger: (...scopes: string[]) => Logger, + ccsEnabled: boolean +): AlertType => { + const logger = getLogger(ALERT_TYPE_CLUSTER_STATE); + return { + id: ALERT_TYPE_CLUSTER_STATE, + name: 'Monitoring Alert - Cluster Status', + actionGroups: [ + { + id: 'default', + name: i18n.translate('xpack.monitoring.alerts.clusterState.actionGroups.default', { + defaultMessage: 'Default', + }), + }, + ], + defaultActionGroupId: 'default', + async executor({ + services, + params, + state, + }: AlertCommonExecutorOptions): Promise { + logger.debug( + `Firing alert with params: ${JSON.stringify(params)} and state: ${JSON.stringify(state)}` + ); + + const preparedAlert = await getPreparedAlert( + ALERT_TYPE_CLUSTER_STATE, + getUiSettingsService, + monitoringCluster, + logger, + ccsEnabled, + services, + fetchClusterState + ); + + if (!preparedAlert) { + return state; + } + + const { emailAddress, data: states, clusters } = preparedAlert; + + const result: AlertCommonState = { ...state }; + const defaultAlertState: AlertClusterStatePerClusterState = { + state: AlertClusterStateState.Green, + ui: { + isFiring: false, + message: null, + severity: 0, + resolvedMS: 0, + triggeredMS: 0, + lastCheckedMS: 0, + }, + }; + + for (const clusterState of states) { + const alertState: AlertClusterStatePerClusterState = + (state[clusterState.clusterUuid] as AlertClusterStatePerClusterState) || + defaultAlertState; + const cluster = clusters.find( + (c: AlertCommonCluster) => c.clusterUuid === clusterState.clusterUuid + ); + if (!cluster) { + logger.warn(`Unable to find cluster for clusterUuid='${clusterState.clusterUuid}'`); + continue; + } + const isNonGreen = clusterState.state !== AlertClusterStateState.Green; + const severity = clusterState.state === AlertClusterStateState.Red ? 2100 : 1100; + + const ui = alertState.ui; + let triggered = ui.triggeredMS; + let resolved = ui.resolvedMS; + let message = ui.message || {}; + let lastState = alertState.state; + const instance = services.alertInstanceFactory(ALERT_TYPE_CLUSTER_STATE); + + if (isNonGreen) { + if (lastState === AlertClusterStateState.Green) { + logger.debug(`Cluster state changed from green to ${clusterState.state}`); + executeActions(instance, cluster, clusterState.state, emailAddress); + lastState = clusterState.state; + triggered = moment().valueOf(); + } + message = getUiMessage(clusterState.state); + resolved = 0; + } else if (!isNonGreen && lastState !== AlertClusterStateState.Green) { + logger.debug(`Cluster state changed from ${lastState} to green`); + executeActions(instance, cluster, clusterState.state, emailAddress, true); + lastState = clusterState.state; + message = getUiMessage(clusterState.state, true); + resolved = moment().valueOf(); + } + + result[clusterState.clusterUuid] = { + state: lastState, + ui: { + message, + isFiring: isNonGreen, + severity, + resolvedMS: resolved, + triggeredMS: triggered, + lastCheckedMS: moment().valueOf(), + }, + } as AlertClusterStatePerClusterState; + } + + return result; + }, + }; +}; diff --git a/x-pack/plugins/monitoring/server/alerts/enums.ts b/x-pack/plugins/monitoring/server/alerts/enums.ts new file mode 100644 index 0000000000000..ccff588743af1 --- /dev/null +++ b/x-pack/plugins/monitoring/server/alerts/enums.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export enum AlertClusterStateState { + Green = 'green', + Red = 'red', + Yellow = 'yellow', +} + +export enum AlertCommonPerClusterMessageTokenType { + Time = 'time', + Link = 'link', +} diff --git a/x-pack/plugins/monitoring/server/alerts/license_expiration.test.ts b/x-pack/plugins/monitoring/server/alerts/license_expiration.test.ts index 0773af6e7f070..92047e300bc1f 100644 --- a/x-pack/plugins/monitoring/server/alerts/license_expiration.test.ts +++ b/x-pack/plugins/monitoring/server/alerts/license_expiration.test.ts @@ -6,42 +6,31 @@ import moment from 'moment-timezone'; import { getLicenseExpiration } from './license_expiration'; -import { - ALERT_TYPE_LICENSE_EXPIRATION, - MONITORING_CONFIG_ALERTING_EMAIL_ADDRESS, -} from '../../common/constants'; +import { ALERT_TYPE_LICENSE_EXPIRATION } from '../../common/constants'; import { Logger } from 'src/core/server'; -import { AlertServices, AlertInstance } from '../../../alerting/server'; +import { AlertServices } from '../../../alerting/server'; import { savedObjectsClientMock } from 'src/core/server/mocks'; import { - AlertState, - AlertClusterState, - AlertParams, - LicenseExpirationAlertExecutorOptions, + AlertCommonParams, + AlertCommonState, + AlertLicensePerClusterState, + AlertLicense, } from './types'; -import { SavedObject, SavedObjectAttributes } from 'src/core/server'; -import { SavedObjectsClientContract } from 'src/core/server'; - -function fillLicense(license: any, clusterUuid?: string) { - return { - hits: { - hits: [ - { - _source: { - license, - cluster_uuid: clusterUuid, - }, - }, - ], - }, - }; -} - -const clusterUuid = 'a4545jhjb'; -const params: AlertParams = { - dateFormat: 'YYYY', - timezone: 'UTC', -}; +import { executeActions } from '../lib/alerts/license_expiration.lib'; +import { PreparedAlert, getPreparedAlert } from '../lib/alerts/get_prepared_alert'; + +jest.mock('../lib/alerts/license_expiration.lib', () => ({ + executeActions: jest.fn(), + getUiMessage: jest.fn(), +})); + +jest.mock('../lib/alerts/get_prepared_alert', () => ({ + getPreparedAlert: jest.fn(() => { + return { + emailAddress: 'foo@foo.com', + }; + }), +})); interface MockServices { callCluster: jest.Mock; @@ -49,428 +38,169 @@ interface MockServices { savedObjectsClient: jest.Mock; } -const alertExecutorOptions: LicenseExpirationAlertExecutorOptions = { - alertId: '', - startedAt: new Date(), - services: { - callCluster: (path: string, opts: any) => new Promise(resolve => resolve()), - alertInstanceFactory: (id: string) => new AlertInstance(), - savedObjectsClient: {} as jest.Mocked, - }, - params: {}, - state: {}, - spaceId: '', - name: '', - tags: [], - previousStartedAt: null, - createdBy: null, - updatedBy: null, -}; - describe('getLicenseExpiration', () => { - const emailAddress = 'foo@foo.com'; - const getUiSettingsService: any = () => ({ - asScopedToClient: (): any => ({ - get: () => new Promise(resolve => resolve(emailAddress)), - }), - }); - const monitoringCluster: any = null; - const logger: Logger = { - warn: jest.fn(), - log: jest.fn(), - debug: jest.fn(), - trace: jest.fn(), - error: jest.fn(), - fatal: jest.fn(), - info: jest.fn(), - get: jest.fn(), + const services: MockServices | AlertServices = { + callCluster: jest.fn(), + alertInstanceFactory: jest.fn(), + savedObjectsClient: savedObjectsClientMock.create(), }; - const getLogger = (): Logger => logger; - const ccrEnabled = false; - afterEach(() => { - (logger.warn as jest.Mock).mockClear(); - }); - - it('should have the right id and actionGroups', () => { - const alert = getLicenseExpiration( - getUiSettingsService, - monitoringCluster, - getLogger, - ccrEnabled - ); - expect(alert.id).toBe(ALERT_TYPE_LICENSE_EXPIRATION); - expect(alert.actionGroups).toEqual([{ id: 'default', name: 'Default' }]); - }); + const params: AlertCommonParams = { + dateFormat: 'YYYY', + timezone: 'UTC', + }; - it('should return the state if no license is provided', async () => { - const alert = getLicenseExpiration( - getUiSettingsService, - monitoringCluster, - getLogger, - ccrEnabled - ); + const emailAddress = 'foo@foo.com'; + const clusterUuid = 'kdksdfj434'; + const clusterName = 'monitoring_test'; + const dateFormat = 'YYYY-MM-DD'; + const cluster = { clusterUuid, clusterName }; + const defaultUiState = { + isFiring: false, + severity: 0, + message: null, + resolvedMS: 0, + lastCheckedMS: 0, + triggeredMS: 0, + }; - const services: MockServices | AlertServices = { - callCluster: jest.fn(), - alertInstanceFactory: jest.fn(), - savedObjectsClient: savedObjectsClientMock.create(), + async function setupAlert( + license: AlertLicense | null, + expiredCheckDateMS: number, + preparedAlertResponse: PreparedAlert | null | undefined = undefined + ): Promise { + const logger: Logger = { + warn: jest.fn(), + log: jest.fn(), + debug: jest.fn(), + trace: jest.fn(), + error: jest.fn(), + fatal: jest.fn(), + info: jest.fn(), + get: jest.fn(), }; - const state = { foo: 1 }; - - const result = await alert.executor({ - ...alertExecutorOptions, - services, - params, - state, - }); - - expect(result).toEqual(state); - }); + const getLogger = (): Logger => logger; + const ccrEnabled = false; + (getPreparedAlert as jest.Mock).mockImplementation(() => { + if (preparedAlertResponse !== undefined) { + return preparedAlertResponse; + } - it('should log a warning if no email is provided', async () => { - const customGetUiSettingsService: any = () => ({ - asScopedToClient: () => ({ - get: () => null, - }), + return { + emailAddress, + data: [license], + clusters: [cluster], + dateFormat, + }; }); - const alert = getLicenseExpiration( - customGetUiSettingsService, - monitoringCluster, - getLogger, - ccrEnabled - ); - const services = { - callCluster: jest.fn( - (method: string, { filterPath }): Promise => { - return new Promise(resolve => { - if (filterPath.includes('hits.hits._source.license.*')) { - resolve( - fillLicense({ - status: 'good', - type: 'basic', - expiry_date_in_millis: moment() - .add(7, 'days') - .valueOf(), - }) - ); - } - resolve({}); - }); - } - ), - alertInstanceFactory: jest.fn(), - savedObjectsClient: savedObjectsClientMock.create(), + const alert = getLicenseExpiration(null as any, null as any, getLogger, ccrEnabled); + const state: AlertCommonState = { + [clusterUuid]: { + expiredCheckDateMS, + ui: { ...defaultUiState }, + } as AlertLicensePerClusterState, }; - const state = {}; + return (await alert.executor({ services, params, state } as any)) as AlertCommonState; + } - await alert.executor({ - ...alertExecutorOptions, - services, - params, - state, - }); - - expect((logger.warn as jest.Mock).mock.calls.length).toBe(1); - expect(logger.warn).toHaveBeenCalledWith( - `Unable to send email for ${ALERT_TYPE_LICENSE_EXPIRATION} because there is no email configured.` - ); + afterEach(() => { + (executeActions as jest.Mock).mockClear(); + (getPreparedAlert as jest.Mock).mockClear(); }); - it('should fire actions if going to expire', async () => { - const scheduleActions = jest.fn(); - const alertInstanceFactory = jest.fn( - (id: string): AlertInstance => { - const instance = new AlertInstance(); - instance.scheduleActions = scheduleActions; - return instance; - } - ); + it('should have the right id and actionGroups', () => { + const alert = getLicenseExpiration(null as any, null as any, jest.fn(), false); + expect(alert.id).toBe(ALERT_TYPE_LICENSE_EXPIRATION); + expect(alert.actionGroups).toEqual([{ id: 'default', name: 'Default' }]); + }); - const alert = getLicenseExpiration( - getUiSettingsService, - monitoringCluster, - getLogger, - ccrEnabled - ); + it('should return the state if no license is provided', async () => { + const result = await setupAlert(null, 0, null); + expect(result[clusterUuid].ui).toEqual(defaultUiState); + }); - const savedObjectsClient = savedObjectsClientMock.create(); - savedObjectsClient.get.mockReturnValue( - new Promise(resolve => { - const savedObject: SavedObject = { - id: '', - type: '', - references: [], - attributes: { - [MONITORING_CONFIG_ALERTING_EMAIL_ADDRESS]: emailAddress, - }, - }; - resolve(savedObject); - }) - ); - const services = { - callCluster: jest.fn( - (method: string, { filterPath }): Promise => { - return new Promise(resolve => { - if (filterPath.includes('hits.hits._source.license.*')) { - resolve( - fillLicense( - { - status: 'active', - type: 'gold', - expiry_date_in_millis: moment() - .add(7, 'days') - .valueOf(), - }, - clusterUuid - ) - ); - } - resolve({}); - }); - } - ), - alertInstanceFactory, - savedObjectsClient, + it('should fire actions if going to expire', async () => { + const expiryDateMS = moment() + .add(7, 'days') + .valueOf(); + const license = { + status: 'active', + type: 'gold', + expiryDateMS, + clusterUuid, }; - - const state = {}; - - const result: AlertState = (await alert.executor({ - ...alertExecutorOptions, - services, - params, - state, - })) as AlertState; - - const newState: AlertClusterState = result[clusterUuid] as AlertClusterState; - + const result = await setupAlert(license, 0); + const newState = result[clusterUuid] as AlertLicensePerClusterState; expect(newState.expiredCheckDateMS > 0).toBe(true); - expect(scheduleActions.mock.calls.length).toBe(1); - expect(scheduleActions.mock.calls[0][1].subject).toBe( - 'NEW X-Pack Monitoring: License Expiration' + expect(executeActions).toHaveBeenCalledWith( + undefined, + cluster, + moment.utc(expiryDateMS), + dateFormat, + emailAddress ); - expect(scheduleActions.mock.calls[0][1].to).toBe(emailAddress); }); it('should fire actions if the user fixed their license', async () => { - const scheduleActions = jest.fn(); - const alertInstanceFactory = jest.fn( - (id: string): AlertInstance => { - const instance = new AlertInstance(); - instance.scheduleActions = scheduleActions; - return instance; - } - ); - const alert = getLicenseExpiration( - getUiSettingsService, - monitoringCluster, - getLogger, - ccrEnabled - ); - - const savedObjectsClient = savedObjectsClientMock.create(); - savedObjectsClient.get.mockReturnValue( - new Promise(resolve => { - const savedObject: SavedObject = { - id: '', - type: '', - references: [], - attributes: { - [MONITORING_CONFIG_ALERTING_EMAIL_ADDRESS]: emailAddress, - }, - }; - resolve(savedObject); - }) - ); - const services = { - callCluster: jest.fn( - (method: string, { filterPath }): Promise => { - return new Promise(resolve => { - if (filterPath.includes('hits.hits._source.license.*')) { - resolve( - fillLicense( - { - status: 'active', - type: 'gold', - expiry_date_in_millis: moment() - .add(120, 'days') - .valueOf(), - }, - clusterUuid - ) - ); - } - resolve({}); - }); - } - ), - alertInstanceFactory, - savedObjectsClient, - }; - - const state: AlertState = { - [clusterUuid]: { - expiredCheckDateMS: moment() - .subtract(1, 'day') - .valueOf(), - ui: { isFiring: true, severity: 0, message: null, resolvedMS: 0, expirationTime: 0 }, - }, + const expiryDateMS = moment() + .add(365, 'days') + .valueOf(); + const license = { + status: 'active', + type: 'gold', + expiryDateMS, + clusterUuid, }; - - const result: AlertState = (await alert.executor({ - ...alertExecutorOptions, - services, - params, - state, - })) as AlertState; - - const newState: AlertClusterState = result[clusterUuid] as AlertClusterState; + const result = await setupAlert(license, 100); + const newState = result[clusterUuid] as AlertLicensePerClusterState; expect(newState.expiredCheckDateMS).toBe(0); - expect(scheduleActions.mock.calls.length).toBe(1); - expect(scheduleActions.mock.calls[0][1].subject).toBe( - 'RESOLVED X-Pack Monitoring: License Expiration' + expect(executeActions).toHaveBeenCalledWith( + undefined, + cluster, + moment.utc(expiryDateMS), + dateFormat, + emailAddress, + true ); - expect(scheduleActions.mock.calls[0][1].to).toBe(emailAddress); }); it('should not fire actions for trial license that expire in more than 14 days', async () => { - const scheduleActions = jest.fn(); - const alertInstanceFactory = jest.fn( - (id: string): AlertInstance => { - const instance = new AlertInstance(); - instance.scheduleActions = scheduleActions; - return instance; - } - ); - const alert = getLicenseExpiration( - getUiSettingsService, - monitoringCluster, - getLogger, - ccrEnabled - ); - - const savedObjectsClient = savedObjectsClientMock.create(); - savedObjectsClient.get.mockReturnValue( - new Promise(resolve => { - const savedObject: SavedObject = { - id: '', - type: '', - references: [], - attributes: { - [MONITORING_CONFIG_ALERTING_EMAIL_ADDRESS]: emailAddress, - }, - }; - resolve(savedObject); - }) - ); - const services = { - callCluster: jest.fn( - (method: string, { filterPath }): Promise => { - return new Promise(resolve => { - if (filterPath.includes('hits.hits._source.license.*')) { - resolve( - fillLicense( - { - status: 'active', - type: 'trial', - expiry_date_in_millis: moment() - .add(15, 'days') - .valueOf(), - }, - clusterUuid - ) - ); - } - resolve({}); - }); - } - ), - alertInstanceFactory, - savedObjectsClient, + const expiryDateMS = moment() + .add(20, 'days') + .valueOf(); + const license = { + status: 'active', + type: 'trial', + expiryDateMS, + clusterUuid, }; - - const state = {}; - const result: AlertState = (await alert.executor({ - ...alertExecutorOptions, - services, - params, - state, - })) as AlertState; - - const newState: AlertClusterState = result[clusterUuid] as AlertClusterState; - expect(newState.expiredCheckDateMS).toBe(undefined); - expect(scheduleActions).not.toHaveBeenCalled(); + const result = await setupAlert(license, 0); + const newState = result[clusterUuid] as AlertLicensePerClusterState; + expect(newState.expiredCheckDateMS).toBe(0); + expect(executeActions).not.toHaveBeenCalled(); }); it('should fire actions for trial license that in 14 days or less', async () => { - const scheduleActions = jest.fn(); - const alertInstanceFactory = jest.fn( - (id: string): AlertInstance => { - const instance = new AlertInstance(); - instance.scheduleActions = scheduleActions; - return instance; - } - ); - const alert = getLicenseExpiration( - getUiSettingsService, - monitoringCluster, - getLogger, - ccrEnabled - ); - - const savedObjectsClient = savedObjectsClientMock.create(); - savedObjectsClient.get.mockReturnValue( - new Promise(resolve => { - const savedObject: SavedObject = { - id: '', - type: '', - references: [], - attributes: { - [MONITORING_CONFIG_ALERTING_EMAIL_ADDRESS]: emailAddress, - }, - }; - resolve(savedObject); - }) - ); - const services = { - callCluster: jest.fn( - (method: string, { filterPath }): Promise => { - return new Promise(resolve => { - if (filterPath.includes('hits.hits._source.license.*')) { - resolve( - fillLicense( - { - status: 'active', - type: 'trial', - expiry_date_in_millis: moment() - .add(13, 'days') - .valueOf(), - }, - clusterUuid - ) - ); - } - resolve({}); - }); - } - ), - alertInstanceFactory, - savedObjectsClient, + const expiryDateMS = moment() + .add(7, 'days') + .valueOf(); + const license = { + status: 'active', + type: 'trial', + expiryDateMS, + clusterUuid, }; - - const state = {}; - const result: AlertState = (await alert.executor({ - ...alertExecutorOptions, - services, - params, - state, - })) as AlertState; - - const newState: AlertClusterState = result[clusterUuid] as AlertClusterState; + const result = await setupAlert(license, 0); + const newState = result[clusterUuid] as AlertLicensePerClusterState; expect(newState.expiredCheckDateMS > 0).toBe(true); - expect(scheduleActions.mock.calls.length).toBe(1); + expect(executeActions).toHaveBeenCalledWith( + undefined, + cluster, + moment.utc(expiryDateMS), + dateFormat, + emailAddress + ); }); }); diff --git a/x-pack/plugins/monitoring/server/alerts/license_expiration.ts b/x-pack/plugins/monitoring/server/alerts/license_expiration.ts index 93397ff3641ae..2e5356150086b 100644 --- a/x-pack/plugins/monitoring/server/alerts/license_expiration.ts +++ b/x-pack/plugins/monitoring/server/alerts/license_expiration.ts @@ -5,24 +5,20 @@ */ import moment from 'moment-timezone'; -import { get } from 'lodash'; import { Logger, ICustomClusterClient, UiSettingsServiceStart } from 'src/core/server'; import { i18n } from '@kbn/i18n'; -import { ALERT_TYPE_LICENSE_EXPIRATION, INDEX_PATTERN_ELASTICSEARCH } from '../../common/constants'; +import { ALERT_TYPE_LICENSE_EXPIRATION } from '../../common/constants'; import { AlertType } from '../../../../plugins/alerting/server'; import { fetchLicenses } from '../lib/alerts/fetch_licenses'; -import { fetchDefaultEmailAddress } from '../lib/alerts/fetch_default_email_address'; -import { fetchClusters } from '../lib/alerts/fetch_clusters'; -import { fetchAvailableCcs } from '../lib/alerts/fetch_available_ccs'; import { - AlertLicense, - AlertState, - AlertClusterState, - AlertClusterUiState, - LicenseExpirationAlertExecutorOptions, + AlertCommonState, + AlertLicensePerClusterState, + AlertCommonExecutorOptions, + AlertCommonCluster, + AlertLicensePerClusterUiState, } from './types'; -import { getCcsIndexPattern } from '../lib/alerts/get_ccs_index_pattern'; import { executeActions, getUiMessage } from '../lib/alerts/license_expiration.lib'; +import { getPreparedAlert } from '../lib/alerts/get_prepared_alert'; const EXPIRES_DAYS = [60, 30, 14, 7]; @@ -32,14 +28,6 @@ export const getLicenseExpiration = ( getLogger: (...scopes: string[]) => Logger, ccsEnabled: boolean ): AlertType => { - async function getCallCluster(services: any): Promise { - if (!monitoringCluster) { - return services.callCluster; - } - - return monitoringCluster.callAsInternalUser; - } - const logger = getLogger(ALERT_TYPE_LICENSE_EXPIRATION); return { id: ALERT_TYPE_LICENSE_EXPIRATION, @@ -53,54 +41,50 @@ export const getLicenseExpiration = ( }, ], defaultActionGroupId: 'default', - async executor({ - services, - params, - state, - }: LicenseExpirationAlertExecutorOptions): Promise { + async executor({ services, params, state }: AlertCommonExecutorOptions): Promise { logger.debug( `Firing alert with params: ${JSON.stringify(params)} and state: ${JSON.stringify(state)}` ); - const callCluster = await getCallCluster(services); - - // Support CCS use cases by querying to find available remote clusters - // and then adding those to the index pattern we are searching against - let esIndexPattern = INDEX_PATTERN_ELASTICSEARCH; - if (ccsEnabled) { - const availableCcs = await fetchAvailableCcs(callCluster); - if (availableCcs.length > 0) { - esIndexPattern = getCcsIndexPattern(esIndexPattern, availableCcs); - } - } - - const clusters = await fetchClusters(callCluster, esIndexPattern); + const preparedAlert = await getPreparedAlert( + ALERT_TYPE_LICENSE_EXPIRATION, + getUiSettingsService, + monitoringCluster, + logger, + ccsEnabled, + services, + fetchLicenses + ); - // Fetch licensing information from cluster_stats documents - const licenses: AlertLicense[] = await fetchLicenses(callCluster, clusters, esIndexPattern); - if (licenses.length === 0) { - logger.warn(`No license found for ${ALERT_TYPE_LICENSE_EXPIRATION}.`); + if (!preparedAlert) { return state; } - const uiSettings = (await getUiSettingsService()).asScopedToClient( - services.savedObjectsClient - ); - const dateFormat: string = await uiSettings.get('dateFormat'); - const timezone: string = await uiSettings.get('dateFormat:tz'); - const emailAddress = await fetchDefaultEmailAddress(uiSettings); - if (!emailAddress) { - // TODO: we can do more here - logger.warn( - `Unable to send email for ${ALERT_TYPE_LICENSE_EXPIRATION} because there is no email configured.` - ); - return; - } + const { emailAddress, data: licenses, clusters, dateFormat } = preparedAlert; - const result: AlertState = { ...state }; + const result: AlertCommonState = { ...state }; + const defaultAlertState: AlertLicensePerClusterState = { + expiredCheckDateMS: 0, + ui: { + isFiring: false, + message: null, + severity: 0, + resolvedMS: 0, + lastCheckedMS: 0, + triggeredMS: 0, + }, + }; for (const license of licenses) { - const licenseState: AlertClusterState = state[license.clusterUuid] || {}; + const alertState: AlertLicensePerClusterState = + (state[license.clusterUuid] as AlertLicensePerClusterState) || defaultAlertState; + const cluster = clusters.find( + (c: AlertCommonCluster) => c.clusterUuid === license.clusterUuid + ); + if (!cluster) { + logger.warn(`Unable to find cluster for clusterUuid='${license.clusterUuid}'`); + continue; + } const $expiry = moment.utc(license.expiryDateMS); let isExpired = false; let severity = 0; @@ -123,31 +107,26 @@ export const getLicenseExpiration = ( } } - const ui: AlertClusterUiState = get(licenseState, 'ui', { - isFiring: false, - message: null, - severity: 0, - resolvedMS: 0, - expirationTime: 0, - }); + const ui = alertState.ui; + let triggered = ui.triggeredMS; let resolved = ui.resolvedMS; let message = ui.message; - let expiredCheckDate = licenseState.expiredCheckDateMS; + let expiredCheckDate = alertState.expiredCheckDateMS; const instance = services.alertInstanceFactory(ALERT_TYPE_LICENSE_EXPIRATION); if (isExpired) { - if (!licenseState.expiredCheckDateMS) { + if (!alertState.expiredCheckDateMS) { logger.debug(`License will expire soon, sending email`); - executeActions(instance, license, $expiry, dateFormat, emailAddress); - expiredCheckDate = moment().valueOf(); + executeActions(instance, cluster, $expiry, dateFormat, emailAddress); + expiredCheckDate = triggered = moment().valueOf(); } - message = getUiMessage(license, timezone); + message = getUiMessage(); resolved = 0; - } else if (!isExpired && licenseState.expiredCheckDateMS) { + } else if (!isExpired && alertState.expiredCheckDateMS) { logger.debug(`License expiration has been resolved, sending email`); - executeActions(instance, license, $expiry, dateFormat, emailAddress, true); + executeActions(instance, cluster, $expiry, dateFormat, emailAddress, true); expiredCheckDate = 0; - message = getUiMessage(license, timezone, true); + message = getUiMessage(true); resolved = moment().valueOf(); } @@ -159,8 +138,10 @@ export const getLicenseExpiration = ( isFiring: expiredCheckDate > 0, severity, resolvedMS: resolved, - }, - }; + triggeredMS: triggered, + lastCheckedMS: moment().valueOf(), + } as AlertLicensePerClusterUiState, + } as AlertLicensePerClusterState; } return result; diff --git a/x-pack/plugins/monitoring/server/alerts/types.d.ts b/x-pack/plugins/monitoring/server/alerts/types.d.ts index ff47d6f2ad4dc..b689d008b51a7 100644 --- a/x-pack/plugins/monitoring/server/alerts/types.d.ts +++ b/x-pack/plugins/monitoring/server/alerts/types.d.ts @@ -5,41 +5,79 @@ */ import { Moment } from 'moment'; import { AlertExecutorOptions } from '../../../alerting/server'; +import { AlertClusterStateState, AlertCommonPerClusterMessageTokenType } from './enums'; export interface AlertLicense { status: string; type: string; expiryDateMS: number; clusterUuid: string; - clusterName: string; } -export interface AlertState { - [clusterUuid: string]: AlertClusterState; +export interface AlertClusterState { + state: AlertClusterStateState; + clusterUuid: string; +} + +export interface AlertCommonState { + [clusterUuid: string]: AlertCommonPerClusterState; } -export interface AlertClusterState { - expiredCheckDateMS: number | Moment; - ui: AlertClusterUiState; +export interface AlertCommonPerClusterState { + ui: AlertCommonPerClusterUiState; } -export interface AlertClusterUiState { +export interface AlertClusterStatePerClusterState extends AlertCommonPerClusterState { + state: AlertClusterStateState; +} + +export interface AlertLicensePerClusterState extends AlertCommonPerClusterState { + expiredCheckDateMS: number; +} + +export interface AlertCommonPerClusterUiState { isFiring: boolean; severity: number; - message: string | null; + message: AlertCommonPerClusterMessage | null; resolvedMS: number; + lastCheckedMS: number; + triggeredMS: number; +} + +export interface AlertCommonPerClusterMessage { + text: string; // Do this. #link this is a link #link + tokens?: AlertCommonPerClusterMessageToken[]; +} + +export interface AlertCommonPerClusterMessageToken { + startToken: string; + endToken?: string; + type: AlertCommonPerClusterMessageTokenType; +} + +export interface AlertCommonPerClusterMessageLinkToken extends AlertCommonPerClusterMessageToken { + url?: string; +} + +export interface AlertCommonPerClusterMessageTimeToken extends AlertCommonPerClusterMessageToken { + isRelative: boolean; + isAbsolute: boolean; +} + +export interface AlertLicensePerClusterUiState extends AlertCommonPerClusterUiState { expirationTime: number; } -export interface AlertCluster { +export interface AlertCommonCluster { clusterUuid: string; + clusterName: string; } -export interface LicenseExpirationAlertExecutorOptions extends AlertExecutorOptions { - state: AlertState; +export interface AlertCommonExecutorOptions extends AlertExecutorOptions { + state: AlertCommonState; } -export interface AlertParams { +export interface AlertCommonParams { dateFormat: string; timezone: string; } diff --git a/x-pack/plugins/monitoring/server/lib/alerts/cluster_state.lib.test.ts b/x-pack/plugins/monitoring/server/lib/alerts/cluster_state.lib.test.ts new file mode 100644 index 0000000000000..81e375734cc50 --- /dev/null +++ b/x-pack/plugins/monitoring/server/lib/alerts/cluster_state.lib.test.ts @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { executeActions, getUiMessage } from './cluster_state.lib'; +import { AlertClusterStateState } from '../../alerts/enums'; +import { AlertCommonPerClusterMessageLinkToken } from '../../alerts/types'; + +describe('clusterState lib', () => { + describe('executeActions', () => { + const clusterName = 'clusterA'; + const instance: any = { scheduleActions: jest.fn() }; + const license: any = { clusterName }; + const status = AlertClusterStateState.Green; + const emailAddress = 'test@test.com'; + + beforeEach(() => { + instance.scheduleActions.mockClear(); + }); + + it('should schedule actions when firing', () => { + executeActions(instance, license, status, emailAddress, false); + expect(instance.scheduleActions).toHaveBeenCalledWith('default', { + subject: 'NEW X-Pack Monitoring: Cluster Status', + message: `Allocate missing replica shards for cluster '${clusterName}'`, + to: emailAddress, + }); + }); + + it('should have a different message for red state', () => { + executeActions(instance, license, AlertClusterStateState.Red, emailAddress, false); + expect(instance.scheduleActions).toHaveBeenCalledWith('default', { + subject: 'NEW X-Pack Monitoring: Cluster Status', + message: `Allocate missing primary and replica shards for cluster '${clusterName}'`, + to: emailAddress, + }); + }); + + it('should schedule actions when resolved', () => { + executeActions(instance, license, status, emailAddress, true); + expect(instance.scheduleActions).toHaveBeenCalledWith('default', { + subject: 'RESOLVED X-Pack Monitoring: Cluster Status', + message: `This cluster alert has been resolved: Allocate missing replica shards for cluster '${clusterName}'`, + to: emailAddress, + }); + }); + }); + + describe('getUiMessage', () => { + it('should return a message when firing', () => { + const message = getUiMessage(AlertClusterStateState.Red, false); + expect(message.text).toBe( + `Elasticsearch cluster status is red. #start_linkAllocate missing primary and replica shards#end_link` + ); + expect(message.tokens && message.tokens.length).toBe(1); + expect(message.tokens && message.tokens[0].startToken).toBe('#start_link'); + expect(message.tokens && message.tokens[0].endToken).toBe('#end_link'); + expect( + message.tokens && (message.tokens[0] as AlertCommonPerClusterMessageLinkToken).url + ).toBe('elasticsearch/indices'); + }); + + it('should return a message when resolved', () => { + const message = getUiMessage(AlertClusterStateState.Green, true); + expect(message.text).toBe(`Elasticsearch cluster status is green.`); + expect(message.tokens).not.toBeDefined(); + }); + }); +}); diff --git a/x-pack/plugins/monitoring/server/lib/alerts/cluster_state.lib.ts b/x-pack/plugins/monitoring/server/lib/alerts/cluster_state.lib.ts new file mode 100644 index 0000000000000..ae66d603507ca --- /dev/null +++ b/x-pack/plugins/monitoring/server/lib/alerts/cluster_state.lib.ts @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { i18n } from '@kbn/i18n'; +import { AlertInstance } from '../../../../alerting/server'; +import { + AlertCommonCluster, + AlertCommonPerClusterMessage, + AlertCommonPerClusterMessageLinkToken, +} from '../../alerts/types'; +import { AlertClusterStateState, AlertCommonPerClusterMessageTokenType } from '../../alerts/enums'; + +const RESOLVED_SUBJECT = i18n.translate('xpack.monitoring.alerts.clusterStatus.resolvedSubject', { + defaultMessage: 'RESOLVED X-Pack Monitoring: Cluster Status', +}); + +const NEW_SUBJECT = i18n.translate('xpack.monitoring.alerts.clusterStatus.newSubject', { + defaultMessage: 'NEW X-Pack Monitoring: Cluster Status', +}); + +const RED_STATUS_MESSAGE = i18n.translate('xpack.monitoring.alerts.clusterStatus.redMessage', { + defaultMessage: 'Allocate missing primary and replica shards', +}); + +const YELLOW_STATUS_MESSAGE = i18n.translate( + 'xpack.monitoring.alerts.clusterStatus.yellowMessage', + { + defaultMessage: 'Allocate missing replica shards', + } +); + +export function executeActions( + instance: AlertInstance, + cluster: AlertCommonCluster, + status: AlertClusterStateState, + emailAddress: string, + resolved: boolean = false +) { + const message = + status === AlertClusterStateState.Red ? RED_STATUS_MESSAGE : YELLOW_STATUS_MESSAGE; + if (resolved) { + instance.scheduleActions('default', { + subject: RESOLVED_SUBJECT, + message: `This cluster alert has been resolved: ${message} for cluster '${cluster.clusterName}'`, + to: emailAddress, + }); + } else { + instance.scheduleActions('default', { + subject: NEW_SUBJECT, + message: `${message} for cluster '${cluster.clusterName}'`, + to: emailAddress, + }); + } +} + +export function getUiMessage( + status: AlertClusterStateState, + resolved: boolean = false +): AlertCommonPerClusterMessage { + if (resolved) { + return { + text: i18n.translate('xpack.monitoring.alerts.clusterStatus.ui.resolvedMessage', { + defaultMessage: `Elasticsearch cluster status is green.`, + }), + }; + } + const message = + status === AlertClusterStateState.Red ? RED_STATUS_MESSAGE : YELLOW_STATUS_MESSAGE; + return { + text: i18n.translate('xpack.monitoring.alerts.clusterStatus.ui.firingMessage', { + defaultMessage: `Elasticsearch cluster status is {status}. #start_link{message}#end_link`, + values: { + status, + message, + }, + }), + tokens: [ + { + startToken: '#start_link', + endToken: '#end_link', + type: AlertCommonPerClusterMessageTokenType.Link, + url: 'elasticsearch/indices', + } as AlertCommonPerClusterMessageLinkToken, + ], + }; +} diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_cluster_state.test.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_cluster_state.test.ts new file mode 100644 index 0000000000000..642ae3c39a027 --- /dev/null +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_cluster_state.test.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { fetchClusterState } from './fetch_cluster_state'; + +describe('fetchClusterState', () => { + it('should return the cluster state', async () => { + const status = 'green'; + const clusterUuid = 'sdfdsaj34434'; + const callCluster = jest.fn(() => ({ + hits: { + hits: [ + { + _source: { + cluster_state: { + status, + }, + cluster_uuid: clusterUuid, + }, + }, + ], + }, + })); + + const clusters = [{ clusterUuid, clusterName: 'foo' }]; + const index = '.monitoring-es-*'; + + const state = await fetchClusterState(callCluster, clusters, index); + expect(state).toEqual([ + { + state: status, + clusterUuid, + }, + ]); + }); +}); diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_cluster_state.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_cluster_state.ts new file mode 100644 index 0000000000000..66ea30d5f2e96 --- /dev/null +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_cluster_state.ts @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { get } from 'lodash'; +import { AlertCommonCluster, AlertClusterState } from '../../alerts/types'; + +export async function fetchClusterState( + callCluster: any, + clusters: AlertCommonCluster[], + index: string +): Promise { + const params = { + index, + filterPath: ['hits.hits._source.cluster_state.status', 'hits.hits._source.cluster_uuid'], + body: { + size: 1, + sort: [{ timestamp: { order: 'desc' } }], + query: { + bool: { + filter: [ + { + terms: { + cluster_uuid: clusters.map(cluster => cluster.clusterUuid), + }, + }, + { + term: { + type: 'cluster_stats', + }, + }, + { + range: { + timestamp: { + gte: 'now-2m', + }, + }, + }, + ], + }, + }, + }, + }; + + const response = await callCluster('search', params); + return get(response, 'hits.hits', []).map((hit: any) => { + return { + state: get(hit, '_source.cluster_state.status'), + clusterUuid: get(hit, '_source.cluster_uuid'), + }; + }); +} diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_clusters.test.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_clusters.test.ts index 78eb9773df15f..7a9b61f37707b 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_clusters.test.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_clusters.test.ts @@ -6,21 +6,51 @@ import { fetchClusters } from './fetch_clusters'; describe('fetchClusters', () => { + const clusterUuid = '1sdfds734'; + const clusterName = 'monitoring'; + it('return a list of clusters', async () => { const callCluster = jest.fn().mockImplementation(() => ({ - aggregations: { - clusters: { - buckets: [ - { - key: 'clusterA', + hits: { + hits: [ + { + _source: { + cluster_uuid: clusterUuid, + cluster_name: clusterName, + }, + }, + ], + }, + })); + const index = '.monitoring-es-*'; + const result = await fetchClusters(callCluster, index); + expect(result).toEqual([{ clusterUuid, clusterName }]); + }); + + it('return the metadata name if available', async () => { + const metadataName = 'custom-monitoring'; + const callCluster = jest.fn().mockImplementation(() => ({ + hits: { + hits: [ + { + _source: { + cluster_uuid: clusterUuid, + cluster_name: clusterName, + cluster_settings: { + cluster: { + metadata: { + display_name: metadataName, + }, + }, + }, }, - ], - }, + }, + ], }, })); const index = '.monitoring-es-*'; const result = await fetchClusters(callCluster, index); - expect(result).toEqual([{ clusterUuid: 'clusterA' }]); + expect(result).toEqual([{ clusterUuid, clusterName: metadataName }]); }); it('should limit the time period in the query', async () => { diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_clusters.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_clusters.ts index 8ef7339618a2c..d1513ac16fb15 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_clusters.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_clusters.ts @@ -4,18 +4,21 @@ * you may not use this file except in compliance with the Elastic License. */ import { get } from 'lodash'; -import { AlertCluster } from '../../alerts/types'; +import { AlertCommonCluster } from '../../alerts/types'; -interface AggregationResult { - key: string; -} - -export async function fetchClusters(callCluster: any, index: string): Promise { +export async function fetchClusters( + callCluster: any, + index: string +): Promise { const params = { index, - filterPath: 'aggregations.clusters.buckets', + filterPath: [ + 'hits.hits._source.cluster_settings.cluster.metadata.display_name', + 'hits.hits._source.cluster_uuid', + 'hits.hits._source.cluster_name', + ], body: { - size: 0, + size: 1000, query: { bool: { filter: [ @@ -34,19 +37,21 @@ export async function fetchClusters(callCluster: any, index: string): Promise ({ - clusterUuid: bucket.key, - })); + return get(response, 'hits.hits', []).map((hit: any) => { + const clusterName: string = + get(hit, '_source.cluster_settings.cluster.metadata.display_name') || + get(hit, '_source.cluster_name') || + get(hit, '_source.cluster_uuid'); + return { + clusterUuid: get(hit, '_source.cluster_uuid'), + clusterName, + }; + }); } diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_licenses.test.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_licenses.test.ts index dd6c074e68b1f..9dcb4ffb82a5f 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_licenses.test.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_licenses.test.ts @@ -6,28 +6,28 @@ import { fetchLicenses } from './fetch_licenses'; describe('fetchLicenses', () => { + const clusterName = 'MyCluster'; + const clusterUuid = 'clusterA'; + const license = { + status: 'active', + expiry_date_in_millis: 1579532493876, + type: 'basic', + }; + it('return a list of licenses', async () => { - const clusterName = 'MyCluster'; - const clusterUuid = 'clusterA'; - const license = { - status: 'active', - expiry_date_in_millis: 1579532493876, - type: 'basic', - }; const callCluster = jest.fn().mockImplementation(() => ({ hits: { hits: [ { _source: { license, - cluster_name: clusterName, cluster_uuid: clusterUuid, }, }, ], }, })); - const clusters = [{ clusterUuid }]; + const clusters = [{ clusterUuid, clusterName }]; const index = '.monitoring-es-*'; const result = await fetchLicenses(callCluster, clusters, index); expect(result).toEqual([ @@ -36,15 +36,13 @@ describe('fetchLicenses', () => { type: license.type, expiryDateMS: license.expiry_date_in_millis, clusterUuid, - clusterName, }, ]); }); it('should only search for the clusters provided', async () => { - const clusterUuid = 'clusterA'; const callCluster = jest.fn(); - const clusters = [{ clusterUuid }]; + const clusters = [{ clusterUuid, clusterName }]; const index = '.monitoring-es-*'; await fetchLicenses(callCluster, clusters, index); const params = callCluster.mock.calls[0][1]; @@ -52,54 +50,11 @@ describe('fetchLicenses', () => { }); it('should limit the time period in the query', async () => { - const clusterUuid = 'clusterA'; const callCluster = jest.fn(); - const clusters = [{ clusterUuid }]; + const clusters = [{ clusterUuid, clusterName }]; const index = '.monitoring-es-*'; await fetchLicenses(callCluster, clusters, index); const params = callCluster.mock.calls[0][1]; expect(params.body.query.bool.filter[2].range.timestamp.gte).toBe('now-2m'); }); - - it('should give priority to the metadata name', async () => { - const clusterName = 'MyCluster'; - const clusterUuid = 'clusterA'; - const license = { - status: 'active', - expiry_date_in_millis: 1579532493876, - type: 'basic', - }; - const callCluster = jest.fn().mockImplementation(() => ({ - hits: { - hits: [ - { - _source: { - license, - cluster_name: 'fakeName', - cluster_uuid: clusterUuid, - cluster_settings: { - cluster: { - metadata: { - display_name: clusterName, - }, - }, - }, - }, - }, - ], - }, - })); - const clusters = [{ clusterUuid }]; - const index = '.monitoring-es-*'; - const result = await fetchLicenses(callCluster, clusters, index); - expect(result).toEqual([ - { - status: license.status, - type: license.type, - expiryDateMS: license.expiry_date_in_millis, - clusterUuid, - clusterName, - }, - ]); - }); }); diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_licenses.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_licenses.ts index 31a68e8aa9c3e..5b05c907e796e 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_licenses.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_licenses.ts @@ -4,21 +4,16 @@ * you may not use this file except in compliance with the Elastic License. */ import { get } from 'lodash'; -import { AlertLicense, AlertCluster } from '../../alerts/types'; +import { AlertLicense, AlertCommonCluster } from '../../alerts/types'; export async function fetchLicenses( callCluster: any, - clusters: AlertCluster[], + clusters: AlertCommonCluster[], index: string ): Promise { const params = { index, - filterPath: [ - 'hits.hits._source.license.*', - 'hits.hits._source.cluster_settings.cluster.metadata.display_name', - 'hits.hits._source.cluster_uuid', - 'hits.hits._source.cluster_name', - ], + filterPath: ['hits.hits._source.license.*', 'hits.hits._source.cluster_uuid'], body: { size: 1, sort: [{ timestamp: { order: 'desc' } }], @@ -50,17 +45,12 @@ export async function fetchLicenses( const response = await callCluster('search', params); return get(response, 'hits.hits', []).map((hit: any) => { - const clusterName: string = - get(hit, '_source.cluster_settings.cluster.metadata.display_name') || - get(hit, '_source.cluster_name') || - get(hit, '_source.cluster_uuid'); const rawLicense: any = get(hit, '_source.license', {}); const license: AlertLicense = { status: rawLicense.status, type: rawLicense.type, expiryDateMS: rawLicense.expiry_date_in_millis, clusterUuid: get(hit, '_source.cluster_uuid'), - clusterName, }; return license; }); diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_status.test.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_status.test.ts new file mode 100644 index 0000000000000..a3bcb61afacd6 --- /dev/null +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_status.test.ts @@ -0,0 +1,122 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { fetchStatus } from './fetch_status'; +import { AlertCommonPerClusterState } from '../../alerts/types'; + +describe('fetchStatus', () => { + const alertType = 'monitoringTest'; + const log = { warn: jest.fn() }; + const start = 0; + const end = 0; + const id = 1; + const defaultUiState = { + isFiring: false, + severity: 0, + message: null, + resolvedMS: 0, + lastCheckedMS: 0, + triggeredMS: 0, + }; + const alertsClient = { + find: jest.fn(() => ({ + total: 1, + data: [ + { + id, + }, + ], + })), + getAlertState: jest.fn(() => ({ + alertTypeState: { + state: { + ui: defaultUiState, + } as AlertCommonPerClusterState, + }, + })), + }; + + afterEach(() => { + (alertsClient.find as jest.Mock).mockClear(); + (alertsClient.getAlertState as jest.Mock).mockClear(); + }); + + it('should fetch from the alerts client', async () => { + const status = await fetchStatus(alertsClient as any, [alertType], start, end, log as any); + expect(status).toEqual([]); + }); + + it('should return alerts that are firing', async () => { + alertsClient.getAlertState = jest.fn(() => ({ + alertTypeState: { + state: { + ui: { + ...defaultUiState, + isFiring: true, + }, + } as AlertCommonPerClusterState, + }, + })); + + const status = await fetchStatus(alertsClient as any, [alertType], start, end, log as any); + expect(status.length).toBe(1); + expect(status[0].type).toBe(alertType); + expect(status[0].isFiring).toBe(true); + }); + + it('should return alerts that have been resolved in the time period', async () => { + alertsClient.getAlertState = jest.fn(() => ({ + alertTypeState: { + state: { + ui: { + ...defaultUiState, + resolvedMS: 1500, + }, + } as AlertCommonPerClusterState, + }, + })); + + const customStart = 1000; + const customEnd = 2000; + + const status = await fetchStatus( + alertsClient as any, + [alertType], + customStart, + customEnd, + log as any + ); + expect(status.length).toBe(1); + expect(status[0].type).toBe(alertType); + expect(status[0].isFiring).toBe(false); + }); + + it('should pass in the right filter to the alerts client', async () => { + await fetchStatus(alertsClient as any, [alertType], start, end, log as any); + expect((alertsClient.find as jest.Mock).mock.calls[0][0].options.filter).toBe( + `alert.attributes.alertTypeId:${alertType}` + ); + }); + + it('should return nothing if no alert state is found', async () => { + alertsClient.getAlertState = jest.fn(() => ({ + alertTypeState: null, + })) as any; + + const status = await fetchStatus(alertsClient as any, [alertType], start, end, log as any); + expect(status).toEqual([]); + }); + + it('should return nothing if no alerts are found', async () => { + alertsClient.find = jest.fn(() => ({ + total: 0, + data: [], + })) as any; + + const status = await fetchStatus(alertsClient as any, [alertType], start, end, log as any); + expect(status).toEqual([]); + }); +}); diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_status.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_status.ts index 9f7c1d5a994d2..bf6ee965d3b2f 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_status.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_status.ts @@ -4,81 +4,53 @@ * you may not use this file except in compliance with the Elastic License. */ import moment from 'moment'; -import { get } from 'lodash'; -import { AlertClusterState } from '../../alerts/types'; -import { ALERT_TYPES, LOGGING_TAG } from '../../../common/constants'; +import { Logger } from '../../../../../../src/core/server'; +import { AlertCommonPerClusterState } from '../../alerts/types'; +import { AlertsClient } from '../../../../alerting/server'; export async function fetchStatus( - callCluster: any, + alertsClient: AlertsClient, + alertTypes: string[], start: number, end: number, - clusterUuid: string, - server: any + log: Logger ): Promise { - // TODO: this shouldn't query task manager directly but rather - // use an api exposed by the alerting/actions plugin - // See https://github.com/elastic/kibana/issues/48442 const statuses = await Promise.all( - ALERT_TYPES.map( + alertTypes.map( type => new Promise(async (resolve, reject) => { - try { - const params = { - index: '.kibana_task_manager', - filterPath: ['hits.hits._source.task.state'], - body: { - size: 1, - sort: [{ updated_at: { order: 'desc' } }], - query: { - bool: { - filter: [ - { - term: { - 'task.taskType': `alerting:${type}`, - }, - }, - ], - }, - }, - }, - }; - - const response = await callCluster('search', params); - const state = get(response, 'hits.hits[0]._source.task.state', '{}'); - const clusterState: AlertClusterState = get( - JSON.parse(state), - `alertTypeState.${clusterUuid}`, - { - expiredCheckDateMS: 0, - ui: { - isFiring: false, - message: null, - severity: 0, - resolvedMS: 0, - expirationTime: 0, - }, - } - ); - const isInBetween = moment(clusterState.ui.resolvedMS).isBetween(start, end); - if (clusterState.ui.isFiring || isInBetween) { - return resolve({ - type, - ...clusterState.ui, - }); - } + // We need to get the id from the alertTypeId + const alerts = await alertsClient.find({ + options: { + filter: `alert.attributes.alertTypeId:${type}`, + }, + }); + if (alerts.total === 0) { return resolve(false); - } catch (err) { - const reason = get(err, 'body.error.type'); - if (reason === 'index_not_found_exception') { - server.log( - ['error', LOGGING_TAG], - `Unable to fetch alerts. Alerts depends on task manager, which has not been started yet.` - ); - } else { - server.log(['error', LOGGING_TAG], err.message); - } + } + + if (alerts.total !== 1) { + log.warn(`Found more than one alert for type ${type} which is unexpected.`); + } + + const id = alerts.data[0].id; + + // Now that we have the id, we can get the state + const states = await alertsClient.getAlertState({ id }); + if (!states || !states.alertTypeState) { + log.warn(`No alert states found for type ${type} which is unexpected.`); return resolve(false); } + + const state = Object.values(states.alertTypeState)[0] as AlertCommonPerClusterState; + const isInBetween = moment(state.ui.resolvedMS).isBetween(start, end); + if (state.ui.isFiring || isInBetween) { + return resolve({ + type, + ...state.ui, + }); + } + return resolve(false); }) ) ); diff --git a/x-pack/plugins/monitoring/server/lib/alerts/get_prepared_alert.test.ts b/x-pack/plugins/monitoring/server/lib/alerts/get_prepared_alert.test.ts new file mode 100644 index 0000000000000..1840a2026a753 --- /dev/null +++ b/x-pack/plugins/monitoring/server/lib/alerts/get_prepared_alert.test.ts @@ -0,0 +1,163 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { getPreparedAlert } from './get_prepared_alert'; +import { fetchClusters } from './fetch_clusters'; +import { fetchDefaultEmailAddress } from './fetch_default_email_address'; + +jest.mock('./fetch_clusters', () => ({ + fetchClusters: jest.fn(), +})); + +jest.mock('./fetch_default_email_address', () => ({ + fetchDefaultEmailAddress: jest.fn(), +})); + +describe('getPreparedAlert', () => { + const uiSettings = { get: jest.fn() }; + const alertType = 'test'; + const getUiSettingsService = async () => ({ + asScopedToClient: () => uiSettings, + }); + const monitoringCluster = null; + const logger = { warn: jest.fn() }; + const ccsEnabled = false; + const services = { + callCluster: jest.fn(), + savedObjectsClient: null, + }; + const emailAddress = 'foo@foo.com'; + const data = [{ foo: 1 }]; + const dataFetcher = () => data; + const clusterName = 'MonitoringCluster'; + const clusterUuid = 'sdf34sdf'; + const clusters = [{ clusterName, clusterUuid }]; + + afterEach(() => { + (uiSettings.get as jest.Mock).mockClear(); + (services.callCluster as jest.Mock).mockClear(); + (fetchClusters as jest.Mock).mockClear(); + (fetchDefaultEmailAddress as jest.Mock).mockClear(); + }); + + beforeEach(() => { + (fetchClusters as jest.Mock).mockImplementation(() => clusters); + (fetchDefaultEmailAddress as jest.Mock).mockImplementation(() => emailAddress); + }); + + it('should return fields as expected', async () => { + (uiSettings.get as jest.Mock).mockImplementation(() => { + return emailAddress; + }); + + const alert = await getPreparedAlert( + alertType, + getUiSettingsService as any, + monitoringCluster as any, + logger as any, + ccsEnabled, + services as any, + dataFetcher as any + ); + + expect(alert && alert.emailAddress).toBe(emailAddress); + expect(alert && alert.data).toBe(data); + }); + + it('should add ccs if specified', async () => { + const ccsClusterName = 'remoteCluster'; + (services.callCluster as jest.Mock).mockImplementation(() => { + return { + [ccsClusterName]: { + connected: true, + }, + }; + }); + + await getPreparedAlert( + alertType, + getUiSettingsService as any, + monitoringCluster as any, + logger as any, + true, + services as any, + dataFetcher as any + ); + + expect((fetchClusters as jest.Mock).mock.calls[0][1].includes(ccsClusterName)).toBe(true); + }); + + it('should ignore ccs if no remote clusters are available', async () => { + const ccsClusterName = 'remoteCluster'; + (services.callCluster as jest.Mock).mockImplementation(() => { + return { + [ccsClusterName]: { + connected: false, + }, + }; + }); + + await getPreparedAlert( + alertType, + getUiSettingsService as any, + monitoringCluster as any, + logger as any, + true, + services as any, + dataFetcher as any + ); + + expect((fetchClusters as jest.Mock).mock.calls[0][1].includes(ccsClusterName)).toBe(false); + }); + + it('should pass in the clusters into the data fetcher', async () => { + const customDataFetcher = jest.fn(() => data); + + await getPreparedAlert( + alertType, + getUiSettingsService as any, + monitoringCluster as any, + logger as any, + true, + services as any, + customDataFetcher as any + ); + + expect((customDataFetcher as jest.Mock).mock.calls[0][1]).toBe(clusters); + }); + + it('should return nothing if the data fetcher returns nothing', async () => { + const customDataFetcher = jest.fn(() => []); + + const result = await getPreparedAlert( + alertType, + getUiSettingsService as any, + monitoringCluster as any, + logger as any, + true, + services as any, + customDataFetcher as any + ); + + expect(result).toBe(null); + }); + + it('should return nothing if there is no email address', async () => { + (fetchDefaultEmailAddress as jest.Mock).mockImplementation(() => null); + + const result = await getPreparedAlert( + alertType, + getUiSettingsService as any, + monitoringCluster as any, + logger as any, + true, + services as any, + dataFetcher as any + ); + + expect(result).toBe(null); + }); +}); diff --git a/x-pack/plugins/monitoring/server/lib/alerts/get_prepared_alert.ts b/x-pack/plugins/monitoring/server/lib/alerts/get_prepared_alert.ts new file mode 100644 index 0000000000000..83a9e26e4c589 --- /dev/null +++ b/x-pack/plugins/monitoring/server/lib/alerts/get_prepared_alert.ts @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Logger, ICustomClusterClient, UiSettingsServiceStart } from 'kibana/server'; +import { CallCluster } from 'src/legacy/core_plugins/elasticsearch'; +import { AlertServices } from '../../../../alerting/server'; +import { AlertCommonCluster } from '../../alerts/types'; +import { INDEX_PATTERN_ELASTICSEARCH } from '../../../common/constants'; +import { fetchAvailableCcs } from './fetch_available_ccs'; +import { getCcsIndexPattern } from './get_ccs_index_pattern'; +import { fetchClusters } from './fetch_clusters'; +import { fetchDefaultEmailAddress } from './fetch_default_email_address'; + +export interface PreparedAlert { + emailAddress: string; + clusters: AlertCommonCluster[]; + data: any[]; + timezone: string; + dateFormat: string; +} + +async function getCallCluster( + monitoringCluster: ICustomClusterClient, + services: Pick +): Promise { + if (!monitoringCluster) { + return services.callCluster; + } + + return monitoringCluster.callAsInternalUser; +} + +export async function getPreparedAlert( + alertType: string, + getUiSettingsService: () => Promise, + monitoringCluster: ICustomClusterClient, + logger: Logger, + ccsEnabled: boolean, + services: Pick, + dataFetcher: ( + callCluster: CallCluster, + clusters: AlertCommonCluster[], + esIndexPattern: string + ) => Promise +): Promise { + const callCluster = await getCallCluster(monitoringCluster, services); + + // Support CCS use cases by querying to find available remote clusters + // and then adding those to the index pattern we are searching against + let esIndexPattern = INDEX_PATTERN_ELASTICSEARCH; + if (ccsEnabled) { + const availableCcs = await fetchAvailableCcs(callCluster); + if (availableCcs.length > 0) { + esIndexPattern = getCcsIndexPattern(esIndexPattern, availableCcs); + } + } + + const clusters = await fetchClusters(callCluster, esIndexPattern); + + // Fetch the specific data + const data = await dataFetcher(callCluster, clusters, esIndexPattern); + if (data.length === 0) { + logger.warn(`No data found for ${alertType}.`); + return null; + } + + const uiSettings = (await getUiSettingsService()).asScopedToClient(services.savedObjectsClient); + const dateFormat: string = await uiSettings.get('dateFormat'); + const timezone: string = await uiSettings.get('dateFormat:tz'); + const emailAddress = await fetchDefaultEmailAddress(uiSettings); + if (!emailAddress) { + // TODO: we can do more here + logger.warn(`Unable to send email for ${alertType} because there is no email configured.`); + return null; + } + + return { + emailAddress, + data, + clusters, + dateFormat, + timezone, + }; +} diff --git a/x-pack/plugins/monitoring/server/lib/alerts/license_expiration.lib.test.ts b/x-pack/plugins/monitoring/server/lib/alerts/license_expiration.lib.test.ts index 1a2eb1e44be84..b99208bdde2c8 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/license_expiration.lib.test.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/license_expiration.lib.test.ts @@ -39,17 +39,26 @@ describe('licenseExpiration lib', () => { }); describe('getUiMessage', () => { - const timezone = 'Europe/London'; - const license: any = { expiryDateMS: moment.tz('2020-01-20 08:00:00', timezone).utc() }; - it('should return a message when firing', () => { - const message = getUiMessage(license, timezone, false); - expect(message).toBe(`This cluster's license is going to expire in #relative at #absolute.`); + const message = getUiMessage(false); + expect(message.text).toBe( + `This cluster's license is going to expire in #relative at #absolute. #start_linkPlease update your license.#end_link` + ); + // LOL How do I avoid this in TS???? + if (!message.tokens) { + return expect(false).toBe(true); + } + expect(message.tokens.length).toBe(3); + expect(message.tokens[0].startToken).toBe('#relative'); + expect(message.tokens[1].startToken).toBe('#absolute'); + expect(message.tokens[2].startToken).toBe('#start_link'); + expect(message.tokens[2].endToken).toBe('#end_link'); }); it('should return a message when resolved', () => { - const message = getUiMessage(license, timezone, true); - expect(message).toBe(`This cluster's license is active.`); + const message = getUiMessage(true); + expect(message.text).toBe(`This cluster's license is active.`); + expect(message.tokens).not.toBeDefined(); }); }); }); diff --git a/x-pack/plugins/monitoring/server/lib/alerts/license_expiration.lib.ts b/x-pack/plugins/monitoring/server/lib/alerts/license_expiration.lib.ts index 41b68d69bbd25..cfe9f02b9bd6a 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/license_expiration.lib.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/license_expiration.lib.ts @@ -6,7 +6,13 @@ import { Moment } from 'moment-timezone'; import { i18n } from '@kbn/i18n'; import { AlertInstance } from '../../../../alerting/server'; -import { AlertLicense } from '../../alerts/types'; +import { + AlertCommonPerClusterMessageLinkToken, + AlertCommonPerClusterMessageTimeToken, + AlertCommonCluster, + AlertCommonPerClusterMessage, +} from '../../alerts/types'; +import { AlertCommonPerClusterMessageTokenType } from '../../alerts/enums'; const RESOLVED_SUBJECT = i18n.translate( 'xpack.monitoring.alerts.licenseExpiration.resolvedSubject', @@ -21,7 +27,7 @@ const NEW_SUBJECT = i18n.translate('xpack.monitoring.alerts.licenseExpiration.ne export function executeActions( instance: AlertInstance, - license: AlertLicense, + cluster: AlertCommonCluster, $expiry: Moment, dateFormat: string, emailAddress: string, @@ -31,14 +37,14 @@ export function executeActions( instance.scheduleActions('default', { subject: RESOLVED_SUBJECT, message: `This cluster alert has been resolved: Cluster '${ - license.clusterName + cluster.clusterName }' license was going to expire on ${$expiry.format(dateFormat)}.`, to: emailAddress, }); } else { instance.scheduleActions('default', { subject: NEW_SUBJECT, - message: `Cluster '${license.clusterName}' license is going to expire on ${$expiry.format( + message: `Cluster '${cluster.clusterName}' license is going to expire on ${$expiry.format( dateFormat )}. Please update your license.`, to: emailAddress, @@ -46,13 +52,37 @@ export function executeActions( } } -export function getUiMessage(license: AlertLicense, timezone: string, resolved: boolean = false) { +export function getUiMessage(resolved: boolean = false): AlertCommonPerClusterMessage { if (resolved) { - return i18n.translate('xpack.monitoring.alerts.licenseExpiration.ui.resolvedMessage', { - defaultMessage: `This cluster's license is active.`, - }); + return { + text: i18n.translate('xpack.monitoring.alerts.licenseExpiration.ui.resolvedMessage', { + defaultMessage: `This cluster's license is active.`, + }), + }; } - return i18n.translate('xpack.monitoring.alerts.licenseExpiration.ui.firingMessage', { - defaultMessage: `This cluster's license is going to expire in #relative at #absolute.`, - }); + return { + text: i18n.translate('xpack.monitoring.alerts.licenseExpiration.ui.firingMessage', { + defaultMessage: `This cluster's license is going to expire in #relative at #absolute. #start_linkPlease update your license.#end_link`, + }), + tokens: [ + { + startToken: '#relative', + type: AlertCommonPerClusterMessageTokenType.Time, + isRelative: true, + isAbsolute: false, + } as AlertCommonPerClusterMessageTimeToken, + { + startToken: '#absolute', + type: AlertCommonPerClusterMessageTokenType.Time, + isAbsolute: true, + isRelative: false, + } as AlertCommonPerClusterMessageTimeToken, + { + startToken: '#start_link', + endToken: '#end_link', + type: AlertCommonPerClusterMessageTokenType.Link, + url: 'license', + } as AlertCommonPerClusterMessageLinkToken, + ], + }; } diff --git a/x-pack/plugins/monitoring/server/lib/cluster/get_clusters_from_request.js b/x-pack/plugins/monitoring/server/lib/cluster/get_clusters_from_request.js index c5091c36c3bbe..1bddede52207b 100644 --- a/x-pack/plugins/monitoring/server/lib/cluster/get_clusters_from_request.js +++ b/x-pack/plugins/monitoring/server/lib/cluster/get_clusters_from_request.js @@ -29,6 +29,7 @@ import { CODE_PATH_BEATS, CODE_PATH_APM, KIBANA_ALERTING_ENABLED, + ALERT_TYPES, } from '../../../common/constants'; import { getApmsForClusters } from '../apm/get_apms_for_clusters'; import { i18n } from '@kbn/i18n'; @@ -102,15 +103,8 @@ export async function getClustersFromRequest( if (isInCodePath(codePaths, [CODE_PATH_ALERTS])) { if (KIBANA_ALERTING_ENABLED) { - const { callWithRequest } = req.server.plugins.elasticsearch.getCluster('monitoring'); - const callCluster = (...args) => callWithRequest(req, ...args); - cluster.alerts = await fetchStatus( - callCluster, - start, - end, - cluster.cluster_uuid, - req.server - ); + const alertsClient = req.getAlertsClient ? req.getAlertsClient() : null; + cluster.alerts = await fetchStatus(alertsClient, ALERT_TYPES, start, end, req.logger); } else { cluster.alerts = await alertsClusterSearch( req, diff --git a/x-pack/plugins/monitoring/server/plugin.ts b/x-pack/plugins/monitoring/server/plugin.ts index 24d8bcaa4397c..784226dca66fe 100644 --- a/x-pack/plugins/monitoring/server/plugin.ts +++ b/x-pack/plugins/monitoring/server/plugin.ts @@ -47,6 +47,7 @@ import { PluginSetupContract as AlertingPluginSetupContract, } from '../../alerting/server'; import { getLicenseExpiration } from './alerts/license_expiration'; +import { getClusterState } from './alerts/cluster_state'; import { InfraPluginSetup } from '../../infra/server'; export interface LegacyAPI { @@ -154,6 +155,17 @@ export class Plugin { config.ui.ccs.enabled ) ); + plugins.alerting.registerType( + getClusterState( + async () => { + const coreStart = (await core.getStartServices())[0]; + return coreStart.uiSettings; + }, + cluster, + this.getLogger, + config.ui.ccs.enabled + ) + ); } // Initialize telemetry diff --git a/x-pack/plugins/monitoring/server/routes/api/v1/alerts/alerts.js b/x-pack/plugins/monitoring/server/routes/api/v1/alerts/alerts.js index 56922bd8e87e2..d5a43d32f600a 100644 --- a/x-pack/plugins/monitoring/server/routes/api/v1/alerts/alerts.js +++ b/x-pack/plugins/monitoring/server/routes/api/v1/alerts/alerts.js @@ -8,8 +8,12 @@ import { schema } from '@kbn/config-schema'; import { isFunction } from 'lodash'; import { ALERT_TYPE_LICENSE_EXPIRATION, + ALERT_TYPE_CLUSTER_STATE, MONITORING_CONFIG_ALERTING_EMAIL_ADDRESS, + ALERT_TYPES, } from '../../../../../common/constants'; +import { handleError } from '../../../../lib/errors'; +import { fetchStatus } from '../../../../lib/alerts/fetch_status'; async function createAlerts(req, alertsClient, { selectedEmailActionId }) { const createdAlerts = []; @@ -17,7 +21,21 @@ async function createAlerts(req, alertsClient, { selectedEmailActionId }) { // Create alerts const ALERT_TYPES = { [ALERT_TYPE_LICENSE_EXPIRATION]: { - schedule: { interval: '10s' }, + schedule: { interval: '1m' }, + actions: [ + { + group: 'default', + id: selectedEmailActionId, + params: { + subject: '{{context.subject}}', + message: `{{context.message}}`, + to: ['{{context.to}}'], + }, + }, + ], + }, + [ALERT_TYPE_CLUSTER_STATE]: { + schedule: { interval: '1m' }, actions: [ { group: 'default', @@ -86,4 +104,37 @@ export function createKibanaAlertsRoute(server) { return { alerts, emailResponse }; }, }); + + server.route({ + method: 'POST', + path: '/api/monitoring/v1/alert_status', + config: { + validate: { + payload: schema.object({ + timeRange: schema.object({ + min: schema.string(), + max: schema.string(), + }), + }), + }, + }, + async handler(req, headers) { + const alertsClient = isFunction(req.getAlertsClient) ? req.getAlertsClient() : null; + if (!alertsClient) { + return headers.response().code(404); + } + + const start = req.payload.timeRange.min; + const end = req.payload.timeRange.max; + let alerts; + + try { + alerts = await fetchStatus(alertsClient, ALERT_TYPES, start, end, req.logger); + } catch (err) { + throw handleError(err, req); + } + + return { alerts }; + }, + }); } diff --git a/x-pack/plugins/remote_clusters/public/application/sections/components/remote_cluster_form/__snapshots__/remote_cluster_form.test.js.snap b/x-pack/plugins/remote_clusters/public/application/sections/components/remote_cluster_form/__snapshots__/remote_cluster_form.test.js.snap index 4c109c557fdb0..1e6c2c4d289aa 100644 --- a/x-pack/plugins/remote_clusters/public/application/sections/components/remote_cluster_form/__snapshots__/remote_cluster_form.test.js.snap +++ b/x-pack/plugins/remote_clusters/public/application/sections/components/remote_cluster_form/__snapshots__/remote_cluster_form.test.js.snap @@ -350,7 +350,7 @@ exports[`RemoteClusterForm proxy mode renders correct connection settings when u description={ @@ -367,7 +367,7 @@ exports[`RemoteClusterForm proxy mode renders correct connection settings when u data-test-subj="remoteClusterFormConnectionModeToggle" label={ @@ -443,11 +443,11 @@ exports[`RemoteClusterForm proxy mode renders correct connection settings when u className="euiTextColor euiTextColor--subdued" > - Use seed nodes by default, or switch to a single proxy address. + Use seed nodes by default, or switch to proxy mode. @@ -534,11 +534,11 @@ exports[`RemoteClusterForm proxy mode renders correct connection settings when u onClick={[Function]} > - Use a single proxy address + Use proxy mode @@ -687,23 +687,37 @@ exports[`RemoteClusterForm proxy mode renders correct connection settings when u + + , + } + } /> } + isInvalid={false} label={ } @@ -711,28 +725,31 @@ exports[`RemoteClusterForm proxy mode renders correct connection settings when u >

@@ -740,20 +757,19 @@ exports[`RemoteClusterForm proxy mode renders correct connection settings when u
-
- + - +
- + + + , + } + } > - The number of socket connections to open per remote cluster. + A string sent in the server_name field of the TLS Server Name Indication extension if TLS is enabled. + + +
@@ -801,37 +845,23 @@ exports[`RemoteClusterForm proxy mode renders correct connection settings when u
- - , - } - } + defaultMessage="The number of socket connections to open per remote cluster." + id="xpack.remoteClusters.remoteClusterForm.fieldSocketConnectionsHelpText" + values={Object {}} /> } - isInvalid={false} label={ } @@ -839,31 +869,28 @@ exports[`RemoteClusterForm proxy mode renders correct connection settings when u >
@@ -871,19 +898,20 @@ exports[`RemoteClusterForm proxy mode renders correct connection settings when u
-
- + - +
- + - - , - } - } + defaultMessage="The number of socket connections to open per remote cluster." + id="xpack.remoteClusters.remoteClusterForm.fieldSocketConnectionsHelpText" + values={Object {}} > - A string sent in the server_name field of the TLS Server Name Indication extension if TLS is enabled. - - - + The number of socket connections to open per remote cluster.
@@ -1447,7 +1447,7 @@ Array [
- Use seed nodes by default, or switch to a single proxy address. + Use seed nodes by default, or switch to proxy mode.
- Use a single proxy address + Use proxy mode
diff --git a/x-pack/plugins/remote_clusters/public/application/sections/components/remote_cluster_form/remote_cluster_form.js b/x-pack/plugins/remote_clusters/public/application/sections/components/remote_cluster_form/remote_cluster_form.js index c98bd73bf83a0..a392cc9607784 100644 --- a/x-pack/plugins/remote_clusters/public/application/sections/components/remote_cluster_form/remote_cluster_form.js +++ b/x-pack/plugins/remote_clusters/public/application/sections/components/remote_cluster_form/remote_cluster_form.js @@ -405,30 +405,6 @@ export class RemoteClusterForm extends Component { /> - - } - helpText={ - - } - fullWidth - > - - this.onFieldsChange({ proxySocketConnections: Number(e.target.value) || null }) - } - fullWidth - /> - + + + } + helpText={ + + } + fullWidth + > + + this.onFieldsChange({ proxySocketConnections: Number(e.target.value) || null }) + } + fullWidth + /> + ); } @@ -498,14 +499,14 @@ export class RemoteClusterForm extends Component { <> } checked={mode === PROXY_MODE} @@ -519,15 +520,38 @@ export class RemoteClusterForm extends Component { <> } - iconType="pin" - size="s" - /> + > + + + + ), + searchString: ( + + + + ), + }} + /> + ) : null} diff --git a/x-pack/plugins/searchprofiler/public/application/editor/editor.tsx b/x-pack/plugins/searchprofiler/public/application/editor/editor.tsx index ece22905c64d9..27f040f3e9eec 100644 --- a/x-pack/plugins/searchprofiler/public/application/editor/editor.tsx +++ b/x-pack/plugins/searchprofiler/public/application/editor/editor.tsx @@ -5,6 +5,8 @@ */ import React, { memo, useRef, useEffect, useState } from 'react'; +import { i18n } from '@kbn/i18n'; +import { EuiScreenReaderOnly } from '@elastic/eui'; import { Editor as AceEditor } from 'brace'; import { initializeEditor } from './init_editor'; @@ -31,6 +33,8 @@ const createEditorShim = (aceEditor: AceEditor) => { }; }; +const EDITOR_INPUT_ID = 'SearchProfilerTextArea'; + export const Editor = memo(({ licenseEnabled, initialValue, onEditorReady }: Props) => { const containerRef = useRef(null as any); const editorInstanceRef = useRef(null as any); @@ -43,10 +47,25 @@ export const Editor = memo(({ licenseEnabled, initialValue, onEditorReady }: Pro const divEl = containerRef.current; editorInstanceRef.current = initializeEditor({ el: divEl, licenseEnabled }); editorInstanceRef.current.setValue(initialValue, 1); + const textarea = divEl.querySelector('textarea'); + if (textarea) { + textarea.setAttribute('id', EDITOR_INPUT_ID); + } setTextArea(licenseEnabled ? containerRef.current!.querySelector('textarea') : null); onEditorReady(createEditorShim(editorInstanceRef.current)); }, [initialValue, onEditorReady, licenseEnabled]); - return
; + return ( + <> + + + +
+ + ); }); diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 79c1bbc49810b..2ae29202ede43 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -4531,7 +4531,6 @@ "xpack.apm.settings.apmIndices.spanIndicesLabel": "スパンインデックス", "xpack.apm.settings.apmIndices.title": "インデックス", "xpack.apm.settings.apmIndices.transactionIndicesLabel": "トランザクションインデックス", - "xpack.apm.settings.customizeUI": "UI をカスタマイズ", "xpack.apm.settings.customizeUI.customLink": "カスタムリンク", "xpack.apm.settings.customizeUI.customLink.create.failed": "リンクを保存できませんでした！", "xpack.apm.settings.customizeUI.customLink.create.failed.message": "リンクを保存するときに問題が発生しました。エラー: 「{errorMessage}」", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 77bf8f1467783..7f5df15dec83a 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -4532,7 +4532,6 @@ "xpack.apm.settings.apmIndices.spanIndicesLabel": "跨度索引", "xpack.apm.settings.apmIndices.title": "索引", "xpack.apm.settings.apmIndices.transactionIndicesLabel": "事务索引", - "xpack.apm.settings.customizeUI": "定制 UI", "xpack.apm.settings.customizeUI.customLink": "定制链接", "xpack.apm.settings.customizeUI.customLink.create.failed": "链接无法保存！", "xpack.apm.settings.customizeUI.customLink.create.failed.message": "保存链接时出现了问题。错误：“{errorMessage}”", diff --git a/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/es_index.test.tsx b/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/es_index.test.tsx index 6ce954f61bcdb..d83396f200829 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/es_index.test.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/es_index.test.tsx @@ -134,7 +134,7 @@ describe('IndexParamsFields renders', () => { ActionParamsProps >; const actionParams = { - documents: ['test'], + documents: [{ test: 123 }], }; const wrapper = mountWithIntl( { .find('[data-test-subj="actionIndexDoc"]') .first() .prop('value') - ).toBe('"test"'); + ).toBe(`{ + "test": 123 +}`); + expect( + wrapper.find('[data-test-subj="indexDocumentAddVariableButton"]').length > 0 + ).toBeTruthy(); }); }); diff --git a/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/es_index.tsx b/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/es_index.tsx index d631882e1f581..56d9f40e40021 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/es_index.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/es_index.tsx @@ -14,6 +14,10 @@ import { EuiSelect, EuiTitle, EuiIconTip, + EuiPopover, + EuiButtonIcon, + EuiContextMenuPanel, + EuiContextMenuItem, } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n/react'; import { i18n } from '@kbn/i18n'; @@ -31,6 +35,7 @@ import { getIndexOptions, getIndexPatterns, } from '../../../common/index_controls'; +import { useXJsonMode } from '../../lib/use_x_json_mode'; export function getActionType(): ActionTypeModel { return { @@ -271,9 +276,29 @@ const IndexParamsFields: React.FunctionComponent { const { documents } = actionParams; - + const { xJsonMode, convertToJson, setXJson, xJson } = useXJsonMode( + documents && documents.length > 0 ? documents[0] : null + ); + const [isVariablesPopoverOpen, setIsVariablesPopoverOpen] = useState(false); + const messageVariablesItems = messageVariables?.map((variable: string, i: number) => ( + { + const value = (xJson ?? '').concat(` {{${variable}}}`); + setXJson(value); + // Keep the documents in sync with the editor content + onDocumentsChange(convertToJson(value)); + setIsVariablesPopoverOpen(false); + }} + > + {`{{${variable}}}`} + + )); function onDocumentsChange(updatedDocuments: string) { try { const documentsJSON = JSON.parse(updatedDocuments); @@ -291,27 +316,55 @@ const IndexParamsFields: React.FunctionComponent setIsVariablesPopoverOpen(true)} + iconType="indexOpen" + title={i18n.translate( + 'xpack.triggersActionsUI.components.builtinActionTypes.indexAction.addVariableTitle', + { + defaultMessage: 'Add variable', + } + )} + aria-label={i18n.translate( + 'xpack.triggersActionsUI.components.builtinActionTypes.indexAction.addVariablePopoverButton', + { + defaultMessage: 'Add variable', + } + )} + /> + } + isOpen={isVariablesPopoverOpen} + closePopover={() => setIsVariablesPopoverOpen(false)} + panelPaddingSize="none" + anchorPosition="downLeft" + > + + + } > 0 ? documents[0] : {}, null, 2)} - onChange={onDocumentsChange} - width="100%" - height="auto" - minLines={6} - maxLines={30} - isReadOnly={false} - setOptions={{ - showLineNumbers: true, - tabSize: 2, - }} - editorProps={{ - $blockScrolling: Infinity, + aria-label={i18n.translate( + 'xpack.triggersActionsUI.components.builtinActionTypes.indexAction.jsonDocAriaLabel', + { + defaultMessage: 'Code editor', + } + )} + value={xJson} + onChange={(xjson: string) => { + setXJson(xjson); + // Keep the documents in sync with the editor content + onDocumentsChange(convertToJson(xjson)); }} - showGutter={true} /> diff --git a/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/types.ts b/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/types.ts index fd35db4304275..84d8b6e8caede 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/types.ts +++ b/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_action_types/types.ts @@ -39,7 +39,7 @@ export interface PagerDutyActionParams { } export interface IndexActionParams { - documents: string[]; + documents: Array>; } export enum ServerLogLevelOptions { diff --git a/x-pack/plugins/triggers_actions_ui/public/application/lib/use_x_json_mode.ts b/x-pack/plugins/triggers_actions_ui/public/application/lib/use_x_json_mode.ts new file mode 100644 index 0000000000000..b677924724ffe --- /dev/null +++ b/x-pack/plugins/triggers_actions_ui/public/application/lib/use_x_json_mode.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { useState } from 'react'; +import { XJsonMode } from '../../../../../../src/plugins/es_ui_shared/console_lang/ace/modes/x_json'; +import { + collapseLiteralStrings, + expandLiteralStrings, +} from '../../../../../../src/plugins/es_ui_shared/console_lang/lib'; +// @ts-ignore +export const xJsonMode = new XJsonMode(); +export const useXJsonMode = (json: Record | null) => { + const [xJson, setXJson] = useState( + json === null ? '' : expandLiteralStrings(JSON.stringify(json, null, 2)) + ); + + return { + xJson, + setXJson, + xJsonMode, + convertToJson: collapseLiteralStrings, + }; +}; diff --git a/x-pack/plugins/uptime/server/lib/adapters/telemetry/kibana_telemetry_adapter.ts b/x-pack/plugins/uptime/server/lib/adapters/telemetry/kibana_telemetry_adapter.ts index 3b2696ba23f7c..fa792429f102e 100644 --- a/x-pack/plugins/uptime/server/lib/adapters/telemetry/kibana_telemetry_adapter.ts +++ b/x-pack/plugins/uptime/server/lib/adapters/telemetry/kibana_telemetry_adapter.ts @@ -5,7 +5,7 @@ */ import moment from 'moment'; -import { ISavedObjectsRepository } from 'kibana/server'; +import { ISavedObjectsRepository, SavedObjectsClientContract } from 'kibana/server'; import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; import { PageViewParams, UptimeTelemetry } from './types'; import { APICaller } from '../framework'; @@ -54,6 +54,9 @@ export class KibanaTelemetryAdapter { } public static countPageView(pageView: PageViewParams) { + if (pageView.refreshTelemetryHistory) { + this.collector = {}; + } const bucketId = this.getBucketToIncrement(); const bucket = this.collector[bucketId]; if (pageView.page === 'Overview') { @@ -94,7 +97,7 @@ export class KibanaTelemetryAdapter { public static async countNoOfUniqueMonitorAndLocations( callCluster: APICaller, - savedObjectsClient: ISavedObjectsRepository + savedObjectsClient: ISavedObjectsRepository | SavedObjectsClientContract ) { const dynamicSettings = await savedObjectsAdapter.getUptimeDynamicSettings(savedObjectsClient); const params = { @@ -161,24 +164,27 @@ export class KibanaTelemetryAdapter { const monitorNameStats: any = result?.aggregations?.monitor_name; const locationNameStats: any = result?.aggregations?.observer_loc_name; const uniqueMonitors: any = result?.aggregations?.monitors.buckets; - const bucket = this.getBucketToIncrement(); - this.collector[bucket].no_of_unique_monitors = numberOfUniqueMonitors; - this.collector[bucket].no_of_unique_observer_locations = numberOfUniqueLocations; - this.collector[bucket].no_of_unique_observer_locations = numberOfUniqueLocations; - this.collector[bucket].monitor_name_stats = { + const bucketId = this.getBucketToIncrement(); + const bucket = this.collector[bucketId]; + + bucket.no_of_unique_monitors = numberOfUniqueMonitors; + bucket.no_of_unique_observer_locations = numberOfUniqueLocations; + bucket.no_of_unique_observer_locations = numberOfUniqueLocations; + bucket.monitor_name_stats = { min_length: monitorNameStats?.min_length ?? 0, max_length: monitorNameStats?.max_length ?? 0, - avg_length: +monitorNameStats?.avg_length.toFixed(2), + avg_length: +(monitorNameStats?.avg_length?.toFixed(2) ?? 0), }; - this.collector[bucket].observer_location_name_stats = { + bucket.observer_location_name_stats = { min_length: locationNameStats?.min_length ?? 0, max_length: locationNameStats?.max_length ?? 0, avg_length: +(locationNameStats?.avg_length?.toFixed(2) ?? 0), }; - this.collector[bucket].monitor_frequency = this.getMonitorsFrequency(uniqueMonitors); + bucket.monitor_frequency = this.getMonitorsFrequency(uniqueMonitors); + return bucket; } private static getMonitorsFrequency(uniqueMonitors = []) { diff --git a/x-pack/plugins/uptime/server/lib/adapters/telemetry/types.ts b/x-pack/plugins/uptime/server/lib/adapters/telemetry/types.ts index 059bea6cc3215..ee3360ecc41b1 100644 --- a/x-pack/plugins/uptime/server/lib/adapters/telemetry/types.ts +++ b/x-pack/plugins/uptime/server/lib/adapters/telemetry/types.ts @@ -10,6 +10,7 @@ export interface PageViewParams { dateEnd: string; autoRefreshEnabled: boolean; autorefreshInterval: number; + refreshTelemetryHistory?: boolean; } export interface Stats { diff --git a/x-pack/plugins/uptime/server/rest_api/create_route_with_auth.ts b/x-pack/plugins/uptime/server/rest_api/create_route_with_auth.ts index 41527d76432cb..966dc20e27a7e 100644 --- a/x-pack/plugins/uptime/server/rest_api/create_route_with_auth.ts +++ b/x-pack/plugins/uptime/server/rest_api/create_route_with_auth.ts @@ -16,7 +16,7 @@ export const createRouteWithAuth = ( const licenseCheckHandler: UMRouteHandler = async (customParams, context, request, response) => { const { statusCode, message } = libs.license(context.licensing.license); if (statusCode === 200) { - return await handler(customParams, context, request, response); + return handler(customParams, context, request, response); } switch (statusCode) { case 400: diff --git a/x-pack/plugins/uptime/server/rest_api/dynamic_settings.ts b/x-pack/plugins/uptime/server/rest_api/dynamic_settings.ts index 2235379ba6f03..3f4e2fc345182 100644 --- a/x-pack/plugins/uptime/server/rest_api/dynamic_settings.ts +++ b/x-pack/plugins/uptime/server/rest_api/dynamic_settings.ts @@ -19,9 +19,6 @@ export const createGetDynamicSettingsRoute: UMRestApiRouteFactory = (libs: UMSer method: 'GET', path: '/api/uptime/dynamic_settings', validate: false, - options: { - tags: ['access:uptime-read'], - }, handler: async ({ dynamicSettings }, _context, _request, response): Promise => { return response.ok({ body: dynamicSettings, @@ -35,6 +32,7 @@ export const createPostDynamicSettingsRoute: UMRestApiRouteFactory = (libs: UMSe validate: { body: schema.object({}, { unknowns: 'allow' }), }, + writeAccess: true, options: { tags: ['access:uptime-write'], }, diff --git a/x-pack/plugins/uptime/server/rest_api/index_state/get_index_pattern.ts b/x-pack/plugins/uptime/server/rest_api/index_state/get_index_pattern.ts index cec5bb1be245f..689a75c5903a6 100644 --- a/x-pack/plugins/uptime/server/rest_api/index_state/get_index_pattern.ts +++ b/x-pack/plugins/uptime/server/rest_api/index_state/get_index_pattern.ts @@ -12,9 +12,6 @@ export const createGetIndexPatternRoute: UMRestApiRouteFactory = (libs: UMServer method: 'GET', path: API_URLS.INDEX_PATTERN, validate: false, - options: { - tags: ['access:uptime-read'], - }, handler: async ({ callES, dynamicSettings }, _context, _request, response): Promise => { try { return response.ok({ diff --git a/x-pack/plugins/uptime/server/rest_api/index_state/get_index_status.ts b/x-pack/plugins/uptime/server/rest_api/index_state/get_index_status.ts index 9c94ef92f9b6e..8ed73d90b2389 100644 --- a/x-pack/plugins/uptime/server/rest_api/index_state/get_index_status.ts +++ b/x-pack/plugins/uptime/server/rest_api/index_state/get_index_status.ts @@ -12,9 +12,6 @@ export const createGetIndexStatusRoute: UMRestApiRouteFactory = (libs: UMServerL method: 'GET', path: API_URLS.INDEX_STATUS, validate: false, - options: { - tags: ['access:uptime-read'], - }, handler: async ({ callES, dynamicSettings }, _context, _request, response): Promise => { try { return response.ok({ diff --git a/x-pack/plugins/uptime/server/rest_api/monitors/index.ts b/x-pack/plugins/uptime/server/rest_api/monitors/index.ts index 7da717b50c149..51b39037049b5 100644 --- a/x-pack/plugins/uptime/server/rest_api/monitors/index.ts +++ b/x-pack/plugins/uptime/server/rest_api/monitors/index.ts @@ -6,4 +6,4 @@ export { createGetMonitorDetailsRoute } from './monitors_details'; export { createGetMonitorLocationsRoute } from './monitor_locations'; -export { createGetStatusBarRoute } from './status'; +export { createGetStatusBarRoute } from './monitor_status'; diff --git a/x-pack/plugins/uptime/server/rest_api/monitors/monitor_locations.ts b/x-pack/plugins/uptime/server/rest_api/monitors/monitor_locations.ts index befa5fd7e0e55..66ce9871506d4 100644 --- a/x-pack/plugins/uptime/server/rest_api/monitors/monitor_locations.ts +++ b/x-pack/plugins/uptime/server/rest_api/monitors/monitor_locations.ts @@ -19,9 +19,6 @@ export const createGetMonitorLocationsRoute: UMRestApiRouteFactory = (libs: UMSe dateEnd: schema.string(), }), }, - options: { - tags: ['access:uptime-read'], - }, handler: async ({ callES, dynamicSettings }, _context, request, response): Promise => { const { monitorId, dateStart, dateEnd } = request.query; diff --git a/x-pack/plugins/uptime/server/rest_api/monitors/status.ts b/x-pack/plugins/uptime/server/rest_api/monitors/monitor_status.ts similarity index 95% rename from x-pack/plugins/uptime/server/rest_api/monitors/status.ts rename to x-pack/plugins/uptime/server/rest_api/monitors/monitor_status.ts index 9bccd64c4bd65..9cf1340fb9409 100644 --- a/x-pack/plugins/uptime/server/rest_api/monitors/status.ts +++ b/x-pack/plugins/uptime/server/rest_api/monitors/monitor_status.ts @@ -20,9 +20,6 @@ export const createGetStatusBarRoute: UMRestApiRouteFactory = (libs: UMServerLib dateEnd: schema.string(), }), }, - options: { - tags: ['access:uptime-read'], - }, handler: async ({ callES, dynamicSettings }, _context, request, response): Promise => { const { monitorId, dateStart, dateEnd } = request.query; const result = await libs.requests.getLatestMonitor({ diff --git a/x-pack/plugins/uptime/server/rest_api/monitors/monitors_details.ts b/x-pack/plugins/uptime/server/rest_api/monitors/monitors_details.ts index b14eb2c138a75..1cc010781457e 100644 --- a/x-pack/plugins/uptime/server/rest_api/monitors/monitors_details.ts +++ b/x-pack/plugins/uptime/server/rest_api/monitors/monitors_details.ts @@ -19,9 +19,6 @@ export const createGetMonitorDetailsRoute: UMRestApiRouteFactory = (libs: UMServ dateEnd: schema.maybe(schema.string()), }), }, - options: { - tags: ['access:uptime-read'], - }, handler: async ({ callES, dynamicSettings }, _context, request, response): Promise => { const { monitorId, dateStart, dateEnd } = request.query; return response.ok({ diff --git a/x-pack/plugins/uptime/server/rest_api/monitors/monitors_durations.ts b/x-pack/plugins/uptime/server/rest_api/monitors/monitors_durations.ts index 10008c4f6c7ea..9743ced13350a 100644 --- a/x-pack/plugins/uptime/server/rest_api/monitors/monitors_durations.ts +++ b/x-pack/plugins/uptime/server/rest_api/monitors/monitors_durations.ts @@ -20,9 +20,6 @@ export const createGetMonitorDurationRoute: UMRestApiRouteFactory = (libs: UMSer dateEnd: schema.string(), }), }, - options: { - tags: ['access:uptime-read'], - }, handler: async ({ callES, dynamicSettings }, _context, request, response): Promise => { const { monitorId, dateStart, dateEnd } = request.query; return response.ok({ diff --git a/x-pack/plugins/uptime/server/rest_api/overview_filters/get_overview_filters.ts b/x-pack/plugins/uptime/server/rest_api/overview_filters/get_overview_filters.ts index 05376f061c05f..deac05f36c8dc 100644 --- a/x-pack/plugins/uptime/server/rest_api/overview_filters/get_overview_filters.ts +++ b/x-pack/plugins/uptime/server/rest_api/overview_filters/get_overview_filters.ts @@ -28,10 +28,6 @@ export const createGetOverviewFilters: UMRestApiRouteFactory = (libs: UMServerLi tags: arrayOrStringType, }), }, - - options: { - tags: ['access:uptime-read'], - }, handler: async ({ callES, dynamicSettings }, _context, request, response) => { const { dateRangeStart, dateRangeEnd, locations, schemes, search, ports, tags } = request.query; diff --git a/x-pack/plugins/uptime/server/rest_api/pings/get_all.ts b/x-pack/plugins/uptime/server/rest_api/pings/get_all.ts index d64c76fc18a80..c76892103da6b 100644 --- a/x-pack/plugins/uptime/server/rest_api/pings/get_all.ts +++ b/x-pack/plugins/uptime/server/rest_api/pings/get_all.ts @@ -23,9 +23,6 @@ export const createGetAllRoute: UMRestApiRouteFactory = (libs: UMServerLibs) => status: schema.maybe(schema.string()), }), }, - options: { - tags: ['access:uptime-read'], - }, handler: async ({ callES, dynamicSettings }, _context, request, response): Promise => { const { dateRangeStart, dateRangeEnd, location, monitorId, size, sort, status } = request.query; diff --git a/x-pack/plugins/uptime/server/rest_api/pings/get_ping_histogram.ts b/x-pack/plugins/uptime/server/rest_api/pings/get_ping_histogram.ts index cbd9ada027b31..dceef5ecb7848 100644 --- a/x-pack/plugins/uptime/server/rest_api/pings/get_ping_histogram.ts +++ b/x-pack/plugins/uptime/server/rest_api/pings/get_ping_histogram.ts @@ -21,9 +21,6 @@ export const createGetPingHistogramRoute: UMRestApiRouteFactory = (libs: UMServe filters: schema.maybe(schema.string()), }), }, - options: { - tags: ['access:uptime-read'], - }, handler: async ({ callES, dynamicSettings }, _context, request, response): Promise => { const { dateStart, dateEnd, statusFilter, monitorId, filters } = request.query; diff --git a/x-pack/plugins/uptime/server/rest_api/pings/get_pings.ts b/x-pack/plugins/uptime/server/rest_api/pings/get_pings.ts index 8129ad70e6d7d..cde9a8c4e47ea 100644 --- a/x-pack/plugins/uptime/server/rest_api/pings/get_pings.ts +++ b/x-pack/plugins/uptime/server/rest_api/pings/get_pings.ts @@ -23,9 +23,6 @@ export const createGetPingsRoute: UMRestApiRouteFactory = (libs: UMServerLibs) = status: schema.maybe(schema.string()), }), }, - options: { - tags: ['access:uptime-read'], - }, handler: async ({ callES, dynamicSettings }, _context, request, response): Promise => { const { dateRangeStart, dateRangeEnd, location, monitorId, size, sort, status } = request.query; diff --git a/x-pack/plugins/uptime/server/rest_api/snapshot/get_snapshot_count.ts b/x-pack/plugins/uptime/server/rest_api/snapshot/get_snapshot_count.ts index 4fda95bbf86da..d870f49280117 100644 --- a/x-pack/plugins/uptime/server/rest_api/snapshot/get_snapshot_count.ts +++ b/x-pack/plugins/uptime/server/rest_api/snapshot/get_snapshot_count.ts @@ -7,7 +7,7 @@ import { schema } from '@kbn/config-schema'; import { UMServerLibs } from '../../lib/lib'; import { UMRestApiRouteFactory } from '../types'; -import { API_URLS } from '../../../../../legacy/plugins/uptime/common/constants/rest_api'; +import { API_URLS } from '../../../../../legacy/plugins/uptime/common/constants'; export const createGetSnapshotCount: UMRestApiRouteFactory = (libs: UMServerLibs) => ({ method: 'GET', @@ -20,9 +20,6 @@ export const createGetSnapshotCount: UMRestApiRouteFactory = (libs: UMServerLibs statusFilter: schema.maybe(schema.string()), }), }, - options: { - tags: ['access:uptime-read'], - }, handler: async ({ callES, dynamicSettings }, _context, request, response): Promise => { const { dateRangeStart, dateRangeEnd, filters, statusFilter } = request.query; const result = await libs.requests.getSnapshotCount({ diff --git a/x-pack/plugins/uptime/server/rest_api/telemetry/log_page_view.ts b/x-pack/plugins/uptime/server/rest_api/telemetry/log_page_view.ts index 1f6f052019870..4b2db71037071 100644 --- a/x-pack/plugins/uptime/server/rest_api/telemetry/log_page_view.ts +++ b/x-pack/plugins/uptime/server/rest_api/telemetry/log_page_view.ts @@ -8,10 +8,11 @@ import { schema } from '@kbn/config-schema'; import { KibanaTelemetryAdapter } from '../../lib/adapters/telemetry'; import { UMRestApiRouteFactory } from '../types'; import { PageViewParams } from '../../lib/adapters/telemetry/types'; +import { API_URLS } from '../../../../../legacy/plugins/uptime/common/constants'; export const createLogPageViewRoute: UMRestApiRouteFactory = () => ({ method: 'POST', - path: '/api/uptime/logPageView', + path: API_URLS.LOG_PAGE_VIEW, validate: { body: schema.object({ page: schema.string(), @@ -19,15 +20,20 @@ export const createLogPageViewRoute: UMRestApiRouteFactory = () => ({ dateEnd: schema.string(), autoRefreshEnabled: schema.boolean(), autorefreshInterval: schema.number(), + refreshTelemetryHistory: schema.maybe(schema.boolean()), }), }, - handler: async ({ callES, dynamicSettings }, _context, _request, response): Promise => { - const result = KibanaTelemetryAdapter.countPageView(_request.body as PageViewParams); + handler: async ( + { savedObjectsClient, callES, dynamicSettings }, + _context, + request, + response + ): Promise => { + await KibanaTelemetryAdapter.countNoOfUniqueMonitorAndLocations(callES, savedObjectsClient); + const pageViewResult = KibanaTelemetryAdapter.countPageView(request.body as PageViewParams); + return response.ok({ - body: result, + body: pageViewResult, }); }, - options: { - tags: ['access:uptime-read'], - }, }); diff --git a/x-pack/plugins/uptime/server/rest_api/types.ts b/x-pack/plugins/uptime/server/rest_api/types.ts index 8bb1e8a6a86c0..aecb099b7bed5 100644 --- a/x-pack/plugins/uptime/server/rest_api/types.ts +++ b/x-pack/plugins/uptime/server/rest_api/types.ts @@ -24,6 +24,7 @@ import { UMServerLibs } from '../lib/lib'; */ export interface UMServerRoute { method: string; + writeAccess?: boolean; handler: T; } diff --git a/x-pack/plugins/uptime/server/rest_api/uptime_route_wrapper.ts b/x-pack/plugins/uptime/server/rest_api/uptime_route_wrapper.ts index 676aced23a25e..7ede9d39c5f2c 100644 --- a/x-pack/plugins/uptime/server/rest_api/uptime_route_wrapper.ts +++ b/x-pack/plugins/uptime/server/rest_api/uptime_route_wrapper.ts @@ -7,21 +7,20 @@ import { UMKibanaRouteWrapper } from './types'; import { savedObjectsAdapter } from '../lib/saved_objects'; -export const uptimeRouteWrapper: UMKibanaRouteWrapper = uptimeRoute => { - return { - ...uptimeRoute, - handler: async (context, request, response) => { - const { callAsCurrentUser: callES } = context.core.elasticsearch.dataClient; - const { client: savedObjectsClient } = context.core.savedObjects; - const dynamicSettings = await savedObjectsAdapter.getUptimeDynamicSettings( - savedObjectsClient - ); - return await uptimeRoute.handler( - { callES, savedObjectsClient, dynamicSettings }, - context, - request, - response - ); - }, - }; -}; +export const uptimeRouteWrapper: UMKibanaRouteWrapper = uptimeRoute => ({ + ...uptimeRoute, + options: { + tags: ['access:uptime-read', ...(uptimeRoute?.writeAccess ? ['access:uptime-write'] : [])], + }, + handler: async (context, request, response) => { + const { callAsCurrentUser: callES } = context.core.elasticsearch.dataClient; + const { client: savedObjectsClient } = context.core.savedObjects; + const dynamicSettings = await savedObjectsAdapter.getUptimeDynamicSettings(savedObjectsClient); + return uptimeRoute.handler( + { callES, savedObjectsClient, dynamicSettings }, + context, + request, + response + ); + }, +}); diff --git a/x-pack/test/api_integration/apis/management/index_lifecycle_management/policies.js b/x-pack/test/api_integration/apis/management/index_lifecycle_management/policies.js index 9cb2e5ee54596..e41dd40bbd417 100644 --- a/x-pack/test/api_integration/apis/management/index_lifecycle_management/policies.js +++ b/x-pack/test/api_integration/apis/management/index_lifecycle_management/policies.js @@ -32,7 +32,9 @@ export default function({ getService }) { after(() => Promise.all([cleanUpEsResources(), cleanUpPolicies()])); describe('list', () => { - it('should have a default policy to manage the Watcher history indices', async () => { + // Disabled as the underline ES API has changed. Need to investigate + // Opened issue: https://github.com/elastic/kibana/issues/62778 + it.skip('should have a default policy to manage the Watcher history indices', async () => { const { body } = await loadPolicies().expect(200); const policy = body.find(policy => policy.name === DEFAULT_POLICY_NAME); diff --git a/x-pack/test/api_integration/apis/ml/index.ts b/x-pack/test/api_integration/apis/ml/index.ts index 4e21faa610bfe..bcba156a64f0e 100644 --- a/x-pack/test/api_integration/apis/ml/index.ts +++ b/x-pack/test/api_integration/apis/ml/index.ts @@ -9,7 +9,10 @@ import { FtrProviderContext } from '../../ftr_provider_context'; export default function({ getService, loadTestFile }: FtrProviderContext) { const ml = getService('ml'); - describe('Machine Learning', function() { + // ML tests need to be disabled in orde to get the ES snapshot with + // https://github.com/elastic/elasticsearch/pull/54713 promoted + // and should be re-enabled as part of https://github.com/elastic/kibana/pull/61980 + describe.skip('Machine Learning', function() { this.tags(['mlqa']); before(async () => { diff --git a/x-pack/test/api_integration/apis/siem/sources.ts b/x-pack/test/api_integration/apis/siem/sources.ts index 0b147022c7cd5..e0db91449a8cc 100644 --- a/x-pack/test/api_integration/apis/siem/sources.ts +++ b/x-pack/test/api_integration/apis/siem/sources.ts @@ -30,7 +30,7 @@ export default function({ getService }: FtrProviderContext) { .then(resp => { const sourceStatus = resp.data.source.status; // test data in x-pack/test/functional/es_archives/auditbeat_test_data/data.json.gz - expect(sourceStatus.indexFields.length).to.be(395); + expect(sourceStatus.indexFields.length).to.be(397); expect(sourceStatus.indicesExist).to.be(true); }); }); diff --git a/x-pack/test/api_integration/apis/uptime/rest/index.ts b/x-pack/test/api_integration/apis/uptime/rest/index.ts index f89bec589847e..9b0cd61c22462 100644 --- a/x-pack/test/api_integration/apis/uptime/rest/index.ts +++ b/x-pack/test/api_integration/apis/uptime/rest/index.ts @@ -42,6 +42,7 @@ export default function({ getService, loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./snapshot')); loadTestFile(require.resolve('./dynamic_settings')); + loadTestFile(require.resolve('./telemetry_collectors')); }); describe('with real-world data', () => { before('load heartbeat data', async () => await esArchiver.load('uptime/full_heartbeat')); diff --git a/x-pack/test/api_integration/apis/uptime/rest/telemetry_collectors.ts b/x-pack/test/api_integration/apis/uptime/rest/telemetry_collectors.ts new file mode 100644 index 0000000000000..b2ec96be0f288 --- /dev/null +++ b/x-pack/test/api_integration/apis/uptime/rest/telemetry_collectors.ts @@ -0,0 +1,129 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../../ftr_provider_context'; +import { API_URLS } from '../../../../../legacy/plugins/uptime/common/constants'; +import { makeChecksWithStatus } from '../graphql/helpers/make_checks'; + +export default function({ getService }: FtrProviderContext) { + const supertest = getService('supertest'); + const es = getService('legacyEs'); + + describe('telemetry collectors', () => { + before('generating data', async () => { + await getService('esArchiver').load('uptime/blank'); + + const observer = { + geo: { + name: 'US-East', + location: '40.7128, -74.0060', + }, + }; + + const observer2 = { + geo: { + name: 'US', + location: '40.7128, -74.0060', + }, + }; + + await makeChecksWithStatus( + es, + 'upMonitorId', + 1, + 1, + 60 * 1000, + { + observer: {}, + monitor: { + name: 'Elastic', + }, + }, + 'up' + ); + + await makeChecksWithStatus( + es, + 'downMonitorId', + 1, + 1, + 120 * 1000, + { + observer, + monitor: { + name: 'Long Name with 22 Char', + }, + }, + 'down' + ); + + await makeChecksWithStatus(es, 'noGeoNameMonitor', 1, 1, 60 * 1000, { observer: {} }, 'down'); + await makeChecksWithStatus( + es, + 'downMonitorId', + 1, + 1, + 1, + { + observer, + monitor: { + name: 'Elastic', + }, + }, + 'down' + ); + + await makeChecksWithStatus(es, 'mixMonitorId', 1, 1, 1, { observer: observer2 }, 'down'); + }); + + after('unload heartbeat index', () => getService('esArchiver').unload('uptime/blank')); + + it('should receive expected results after calling monitor/overview logging', async () => { + // call monitor page + await supertest + .post(API_URLS.LOG_PAGE_VIEW) + .set('kbn-xsrf', 'true') + .send({ + page: 'Monitor', + autorefreshInterval: 100, + dateStart: 'now/d', + dateEnd: 'now/d', + autoRefreshEnabled: true, + refreshTelemetryHistory: true, + }) + .expect(200); + + // call overview page + const { body: result } = await supertest + .post(API_URLS.LOG_PAGE_VIEW) + .set('kbn-xsrf', 'true') + .send({ + page: 'Overview', + autorefreshInterval: 60, + dateStart: 'now/d', + dateEnd: 'now-30', + autoRefreshEnabled: true, + }) + .expect(200); + + expect(result).to.eql({ + overview_page: 1, + monitor_page: 1, + no_of_unique_monitors: 4, + settings_page: 0, + monitor_frequency: [120, 0.001, 60, 60], + monitor_name_stats: { min_length: 7, max_length: 22, avg_length: 12 }, + no_of_unique_observer_locations: 3, + observer_location_name_stats: { min_length: 2, max_length: 7, avg_length: 4.8 }, + dateRangeStart: ['now/d', 'now/d'], + dateRangeEnd: ['now/d', 'now-30'], + autoRefreshEnabled: true, + autorefreshInterval: [100, 60], + }); + }); + }); +} diff --git a/x-pack/test/api_integration/apis/uptime/telemetry_collectors.ts b/x-pack/test/api_integration/apis/uptime/telemetry_collectors.ts deleted file mode 100644 index e33c6120557bb..0000000000000 --- a/x-pack/test/api_integration/apis/uptime/telemetry_collectors.ts +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { FtrProviderContext } from '../../ftr_provider_context'; - -export default function({ getService }: FtrProviderContext) { - const supertest = getService('supertest'); - - describe('telemetry collectors', () => { - it('should receive a 200 for overview page logging', async () => { - await supertest.get('/api/uptime/logOverview').expect(200); - }); - - it('should receive a 200 for monitor page logging', async () => { - await supertest.get('/api/uptime/logMonitor').expect(200); - }); - }); -} diff --git a/x-pack/test/functional/apps/machine_learning/index.ts b/x-pack/test/functional/apps/machine_learning/index.ts index 47c699e309491..143899a61ffb9 100644 --- a/x-pack/test/functional/apps/machine_learning/index.ts +++ b/x-pack/test/functional/apps/machine_learning/index.ts @@ -8,7 +8,10 @@ import { FtrProviderContext } from '../../ftr_provider_context'; export default function({ getService, loadTestFile }: FtrProviderContext) { const ml = getService('ml'); - describe('machine learning', function() { + // ML tests need to be disabled in orde to get the ES snapshot with + // https://github.com/elastic/elasticsearch/pull/54713 promoted + // and should be re-enabled as part of https://github.com/elastic/kibana/pull/61980 + describe.skip('machine learning', function() { this.tags('ciGroup3'); before(async () => {