diff --git a/x-pack/legacy/plugins/infra/common/ecs_allowed_list.test.ts b/x-pack/legacy/plugins/infra/common/ecs_allowed_list.test.ts new file mode 100644 index 0000000000000..66ed681255d34 --- /dev/null +++ b/x-pack/legacy/plugins/infra/common/ecs_allowed_list.test.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + getAllowedListForPrefix, + ECS_ALLOWED_LIST, + K8S_ALLOWED_LIST, + PROMETHEUS_ALLOWED_LIST, + DOCKER_ALLOWED_LIST, +} from './ecs_allowed_list'; +describe('getAllowedListForPrefix()', () => { + test('kubernetes', () => { + expect(getAllowedListForPrefix('kubernetes.pod')).toEqual([ + ...ECS_ALLOWED_LIST, + 'kubernetes.pod', + ...K8S_ALLOWED_LIST, + ]); + }); + test('docker', () => { + expect(getAllowedListForPrefix('docker.container')).toEqual([ + ...ECS_ALLOWED_LIST, + 'docker.container', + ...DOCKER_ALLOWED_LIST, + ]); + }); + test('prometheus', () => { + expect(getAllowedListForPrefix('prometheus.metrics')).toEqual([ + ...ECS_ALLOWED_LIST, + 'prometheus.metrics', + ...PROMETHEUS_ALLOWED_LIST, + ]); + }); + test('anything.else', () => { + expect(getAllowedListForPrefix('anything.else')).toEqual([ + ...ECS_ALLOWED_LIST, + 'anything.else', + ]); + }); +}); diff --git a/x-pack/legacy/plugins/infra/common/ecs_allowed_list.ts b/x-pack/legacy/plugins/infra/common/ecs_allowed_list.ts new file mode 100644 index 0000000000000..37b037214bb02 --- /dev/null +++ b/x-pack/legacy/plugins/infra/common/ecs_allowed_list.ts @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { first } from 'lodash'; + +export const ECS_ALLOWED_LIST = [ + 'host', + 'cloud', + 'event', + 'agent', + 'fields', + 'service', + 'ecs', + 'metricset', + 'tags', + 'message', + 'labels', + '@timestamp', + 'source', + 'container', +]; + +export const K8S_ALLOWED_LIST = [ + 'kubernetes.pod.name', + 'kubernetes.pod.uid', + 'kubernetes.namespace', + 'kubernetes.node.name', + 'kubernetes.labels', + 'kubernetes.annotations', + 'kubernetes.replicaset.name', + 'kubernetes.deployment.name', + 'kubernetes.statefulset.name', + 'kubernetes.container.name', + 'kubernetes.container.image', +]; + +export const PROMETHEUS_ALLOWED_LIST = ['prometheus.labels', 'prometheus.metrics']; + +export const DOCKER_ALLOWED_LIST = [ + 'docker.container.id', + 'docker.container.image', + 'docker.container.name', + 'docker.container.labels', +]; + +export const getAllowedListForPrefix = (prefix: string) => { + const firstPart = first(prefix.split(/\./)); + const defaultAllowedList = prefix ? [...ECS_ALLOWED_LIST, prefix] : ECS_ALLOWED_LIST; + switch (firstPart) { + case 'docker': + return [...defaultAllowedList, ...DOCKER_ALLOWED_LIST]; + case 'prometheus': + return [...defaultAllowedList, ...PROMETHEUS_ALLOWED_LIST]; + case 'kubernetes': + return [...defaultAllowedList, ...K8S_ALLOWED_LIST]; + default: + return defaultAllowedList; + } +}; diff --git a/x-pack/legacy/plugins/infra/common/graphql/types.ts b/x-pack/legacy/plugins/infra/common/graphql/types.ts index 13771a1e09ce7..89d5608ad7c29 100644 --- a/x-pack/legacy/plugins/infra/common/graphql/types.ts +++ b/x-pack/legacy/plugins/infra/common/graphql/types.ts @@ -131,6 +131,8 @@ export interface InfraIndexField { searchable: boolean; /** Whether the field's values can be aggregated */ aggregatable: boolean; + /** Whether the field should be displayed based on event.module and a ECS allowed list */ + displayable: boolean; } /** A consecutive sequence of log entries */ export interface InfraLogEntryInterval { @@ -1138,6 +1140,8 @@ export namespace SourceStatusFields { searchable: boolean; aggregatable: boolean; + + displayable: boolean; }; } diff --git a/x-pack/legacy/plugins/infra/public/components/metrics_explorer/group_by.tsx b/x-pack/legacy/plugins/infra/public/components/metrics_explorer/group_by.tsx index 77be28f8ca6ec..0a126297a9d26 100644 --- a/x-pack/legacy/plugins/infra/public/components/metrics_explorer/group_by.tsx +++ b/x-pack/legacy/plugins/infra/public/components/metrics_explorer/group_by.tsx @@ -9,6 +9,7 @@ import { InjectedIntl, injectI18n } from '@kbn/i18n/react'; import React, { useCallback } from 'react'; import { FieldType } from 'ui/index_patterns'; import { MetricsExplorerOptions } from '../../containers/metrics_explorer/use_metrics_explorer_options'; +import { isDisplayable } from '../../utils/is_displayable'; interface Props { intl: InjectedIntl; @@ -25,6 +26,19 @@ export const MetricsExplorerGroupBy = injectI18n(({ intl, options, onChange, fie }, [onChange] ); + + const metricPrefixes = options.metrics + .map( + metric => + (metric.field && + metric.field + .split(/\./) + .slice(0, 2) + .join('.')) || + null + ) + .filter(metric => metric) as string[]; + return ( f.aggregatable && f.type === 'string') + .filter(f => isDisplayable(f, metricPrefixes) && f.aggregatable && f.type === 'string') .map(f => ({ label: f.name }))} onChange={handleChange} isClearable={true} diff --git a/x-pack/legacy/plugins/infra/public/components/metrics_explorer/kuery_bar.tsx b/x-pack/legacy/plugins/infra/public/components/metrics_explorer/kuery_bar.tsx index 48d57f682a11b..2f2da4d2a260c 100644 --- a/x-pack/legacy/plugins/infra/public/components/metrics_explorer/kuery_bar.tsx +++ b/x-pack/legacy/plugins/infra/public/components/metrics_explorer/kuery_bar.tsx @@ -10,6 +10,7 @@ import React, { useEffect, useState } from 'react'; import { StaticIndexPattern } from 'ui/index_patterns'; import { WithKueryAutocompletion } from '../../containers/with_kuery_autocompletion'; import { AutocompleteField } from '../autocomplete_field'; +import { isDisplayable } from '../../utils/is_displayable'; interface Props { intl: InjectedIntl; @@ -44,8 +45,13 @@ export const MetricsExplorerKueryBar = injectI18n( setDraftQuery(query); }; + const filteredDerivedIndexPattern = { + ...derivedIndexPattern, + fields: derivedIndexPattern.fields.filter(field => isDisplayable(field)), + }; + return ( - + {({ isLoadingSuggestions, loadSuggestions, suggestions }) => ( ({ label: field.name, value: field.name })); + const comboOptions = fields + .filter(field => isDisplayable(field)) + .map(field => ({ label: field.name, value: field.name })); const selectedOptions = options.metrics .filter(m => m.aggregation !== MetricsExplorerAggregation.count) .map(metric => ({ diff --git a/x-pack/legacy/plugins/infra/public/containers/source/source_fields_fragment.gql_query.ts b/x-pack/legacy/plugins/infra/public/containers/source/source_fields_fragment.gql_query.ts index f1f036b8c1ed7..0c28220aed802 100644 --- a/x-pack/legacy/plugins/infra/public/containers/source/source_fields_fragment.gql_query.ts +++ b/x-pack/legacy/plugins/infra/public/containers/source/source_fields_fragment.gql_query.ts @@ -48,6 +48,7 @@ export const sourceStatusFieldsFragment = gql` type searchable aggregatable + displayable } logIndicesExist metricIndicesExist diff --git a/x-pack/legacy/plugins/infra/public/graphql/introspection.json b/x-pack/legacy/plugins/infra/public/graphql/introspection.json index e02805b2acef2..d1c1c197d160f 100644 --- a/x-pack/legacy/plugins/infra/public/graphql/introspection.json +++ b/x-pack/legacy/plugins/infra/public/graphql/introspection.json @@ -1125,6 +1125,18 @@ }, "isDeprecated": false, "deprecationReason": null + }, + { + "name": "displayable", + "description": "Whether the field should be displayed based on event.module and a ECS allowed list", + "args": [], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { "kind": "SCALAR", "name": "Boolean", "ofType": null } + }, + "isDeprecated": false, + "deprecationReason": null } ], "inputFields": null, diff --git a/x-pack/legacy/plugins/infra/public/graphql/types.ts b/x-pack/legacy/plugins/infra/public/graphql/types.ts index 13771a1e09ce7..89d5608ad7c29 100644 --- a/x-pack/legacy/plugins/infra/public/graphql/types.ts +++ b/x-pack/legacy/plugins/infra/public/graphql/types.ts @@ -131,6 +131,8 @@ export interface InfraIndexField { searchable: boolean; /** Whether the field's values can be aggregated */ aggregatable: boolean; + /** Whether the field should be displayed based on event.module and a ECS allowed list */ + displayable: boolean; } /** A consecutive sequence of log entries */ export interface InfraLogEntryInterval { @@ -1138,6 +1140,8 @@ export namespace SourceStatusFields { searchable: boolean; aggregatable: boolean; + + displayable: boolean; }; } diff --git a/x-pack/legacy/plugins/infra/public/utils/is_displayable.test.ts b/x-pack/legacy/plugins/infra/public/utils/is_displayable.test.ts new file mode 100644 index 0000000000000..ebd5c07327e9b --- /dev/null +++ b/x-pack/legacy/plugins/infra/public/utils/is_displayable.test.ts @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { isDisplayable } from './is_displayable'; + +describe('isDisplayable()', () => { + test('field that is not displayable', () => { + const field = { + name: 'some.field', + type: 'number', + displayable: false, + }; + expect(isDisplayable(field)).toBe(false); + }); + test('field that is displayable', () => { + const field = { + name: 'some.field', + type: 'number', + displayable: true, + }; + expect(isDisplayable(field)).toBe(true); + }); + test('field that an ecs field', () => { + const field = { + name: '@timestamp', + type: 'date', + displayable: true, + }; + expect(isDisplayable(field)).toBe(true); + }); + test('field that matches same prefix', () => { + const field = { + name: 'system.network.name', + type: 'string', + displayable: true, + }; + expect(isDisplayable(field, ['system.network'])).toBe(true); + }); + test('field that does not matches same prefix', () => { + const field = { + name: 'system.load.1', + type: 'number', + displayable: true, + }; + expect(isDisplayable(field, ['system.network'])).toBe(false); + }); + test('field that is an K8s allowed field but does not match prefix', () => { + const field = { + name: 'kubernetes.namespace', + type: 'string', + displayable: true, + }; + expect(isDisplayable(field, ['kubernetes.pod'])).toBe(true); + }); + test('field that is a Prometheus allowed field but does not match prefix', () => { + const field = { + name: 'prometheus.labels.foo.bar', + type: 'string', + displayable: true, + }; + expect(isDisplayable(field, ['prometheus.metrics'])).toBe(true); + }); +}); diff --git a/x-pack/legacy/plugins/infra/public/utils/is_displayable.ts b/x-pack/legacy/plugins/infra/public/utils/is_displayable.ts new file mode 100644 index 0000000000000..f79ce792cac3d --- /dev/null +++ b/x-pack/legacy/plugins/infra/public/utils/is_displayable.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { FieldType } from 'ui/index_patterns'; +import { startsWith, uniq } from 'lodash'; +import { getAllowedListForPrefix } from '../../common/ecs_allowed_list'; + +interface DisplayableFieldType extends FieldType { + displayable?: boolean; +} + +const fieldStartsWith = (field: DisplayableFieldType) => (name: string) => + startsWith(field.name, name); + +export const isDisplayable = (field: DisplayableFieldType, additionalPrefixes: string[] = []) => { + // We need to start with at least one prefix, even if it's empty + const prefixes = additionalPrefixes && additionalPrefixes.length ? additionalPrefixes : ['']; + // Create a set of allowed list based on the prefixes + const allowedList = prefixes.reduce( + (acc, prefix) => { + return uniq([...acc, ...getAllowedListForPrefix(prefix)]); + }, + [] as string[] + ); + // If the field is displayable and part of the allowed list or covered by the prefix + return ( + (field.displayable && prefixes.some(fieldStartsWith(field))) || + allowedList.some(fieldStartsWith(field)) + ); +}; diff --git a/x-pack/legacy/plugins/infra/server/graphql/source_status/schema.gql.ts b/x-pack/legacy/plugins/infra/server/graphql/source_status/schema.gql.ts index 10acb9650b108..e0482382c6d6a 100644 --- a/x-pack/legacy/plugins/infra/server/graphql/source_status/schema.gql.ts +++ b/x-pack/legacy/plugins/infra/server/graphql/source_status/schema.gql.ts @@ -17,6 +17,8 @@ export const sourceStatusSchema = gql` searchable: Boolean! "Whether the field's values can be aggregated" aggregatable: Boolean! + "Whether the field should be displayed based on event.module and a ECS allowed list" + displayable: Boolean! } extend type InfraSourceStatus { diff --git a/x-pack/legacy/plugins/infra/server/graphql/types.ts b/x-pack/legacy/plugins/infra/server/graphql/types.ts index 98b046c8d0c65..dee4d569ec733 100644 --- a/x-pack/legacy/plugins/infra/server/graphql/types.ts +++ b/x-pack/legacy/plugins/infra/server/graphql/types.ts @@ -159,6 +159,8 @@ export interface InfraIndexField { searchable: boolean; /** Whether the field's values can be aggregated */ aggregatable: boolean; + /** Whether the field should be displayed based on event.module and a ECS allowed list */ + displayable: boolean; } /** A consecutive sequence of log entries */ export interface InfraLogEntryInterval { @@ -1123,6 +1125,8 @@ export namespace InfraIndexFieldResolvers { searchable?: SearchableResolver; /** Whether the field's values can be aggregated */ aggregatable?: AggregatableResolver; + /** Whether the field should be displayed based on event.module and a ECS allowed list */ + displayable?: DisplayableResolver; } export type NameResolver = Resolver< @@ -1145,6 +1149,11 @@ export namespace InfraIndexFieldResolvers { Parent = InfraIndexField, Context = InfraContext > = Resolver; + export type DisplayableResolver< + R = boolean, + Parent = InfraIndexField, + Context = InfraContext + > = Resolver; } /** A consecutive sequence of log entries */ export namespace InfraLogEntryIntervalResolvers { diff --git a/x-pack/legacy/plugins/infra/server/lib/adapters/fields/adapter_types.ts b/x-pack/legacy/plugins/infra/server/lib/adapters/fields/adapter_types.ts index 1116cb9969f2b..8f12c0f22a483 100644 --- a/x-pack/legacy/plugins/infra/server/lib/adapters/fields/adapter_types.ts +++ b/x-pack/legacy/plugins/infra/server/lib/adapters/fields/adapter_types.ts @@ -15,4 +15,5 @@ export interface IndexFieldDescriptor { type: string; searchable: boolean; aggregatable: boolean; + displayable: boolean; } diff --git a/x-pack/legacy/plugins/infra/server/lib/adapters/fields/framework_fields_adapter.ts b/x-pack/legacy/plugins/infra/server/lib/adapters/fields/framework_fields_adapter.ts index ee92fb88e7969..2cea001d87b00 100644 --- a/x-pack/legacy/plugins/infra/server/lib/adapters/fields/framework_fields_adapter.ts +++ b/x-pack/legacy/plugins/infra/server/lib/adapters/fields/framework_fields_adapter.ts @@ -4,8 +4,24 @@ * you may not use this file except in compliance with the Elastic License. */ +import { startsWith, uniq } from 'lodash'; import { InfraBackendFrameworkAdapter, InfraFrameworkRequest } from '../framework'; import { FieldsAdapter, IndexFieldDescriptor } from './adapter_types'; +import { getAllowedListForPrefix } from '../../../../common/ecs_allowed_list'; + +interface Bucket { + key: string; + doc_count: number; +} + +interface DataSetResponse { + modules: { + buckets: Bucket[]; + }; + dataSets: { + buckets: Bucket[]; + }; +} export class FrameworkFieldsAdapter implements FieldsAdapter { private framework: InfraBackendFrameworkAdapter; @@ -22,6 +38,55 @@ export class FrameworkFieldsAdapter implements FieldsAdapter { const response = await indexPatternsService.getFieldsForWildcard({ pattern: indices, }); - return response; + const { dataSets, modules } = await this.getDataSetsAndModules(request, indices); + const allowedList = modules.reduce( + (acc, name) => uniq([...acc, ...getAllowedListForPrefix(name)]), + [] as string[] + ); + const dataSetsWithAllowedList = [...allowedList, ...dataSets]; + return response.map(field => ({ + ...field, + displayable: dataSetsWithAllowedList.some(name => startsWith(field.name, name)), + })); + } + + private async getDataSetsAndModules( + request: InfraFrameworkRequest, + indices: string + ): Promise<{ dataSets: string[]; modules: string[] }> { + const params = { + index: indices, + allowNoIndices: true, + ignoreUnavailable: true, + body: { + aggs: { + modules: { + terms: { + field: 'event.modules', + size: 1000, + }, + }, + dataSets: { + terms: { + field: 'event.dataset', + size: 1000, + }, + }, + }, + }, + }; + const response = await this.framework.callWithRequest<{}, DataSetResponse>( + request, + 'search', + params + ); + if (!response.aggregations) { + return { dataSets: [], modules: [] }; + } + const { modules, dataSets } = response.aggregations; + return { + modules: modules.buckets.map(bucket => bucket.key), + dataSets: dataSets.buckets.map(bucket => bucket.key), + }; } }