diff --git a/packages/kbn-securitysolution-grouping/src/containers/query/types.ts b/packages/kbn-securitysolution-grouping/src/containers/query/types.ts index 095bf90ea01e8..b208aef1e840b 100644 --- a/packages/kbn-securitysolution-grouping/src/containers/query/types.ts +++ b/packages/kbn-securitysolution-grouping/src/containers/query/types.ts @@ -11,9 +11,14 @@ import type { MappingRuntimeField, MappingRuntimeFields, } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { RuntimeFieldSpec, RuntimePrimitiveTypes } from '@kbn/data-views-plugin/common'; import type { BoolQuery } from '@kbn/es-query'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +type RunTimeMappings = + | Record & { type: RuntimePrimitiveTypes }> + | undefined; + interface BoolAgg { bool: BoolQuery; } @@ -29,7 +34,7 @@ export interface GroupingQueryArgs { from: string; groupByField: string; rootAggregations?: NamedAggregation[]; - runtimeMappings?: MappingRuntimeFields; + runtimeMappings?: RunTimeMappings; additionalAggregationsRoot?: NamedAggregation[]; pageNumber?: number; uniqueValue: string; diff --git a/x-pack/plugins/security_solution/public/common/components/events_viewer/use_timelines_events.tsx b/x-pack/plugins/security_solution/public/common/components/events_viewer/use_timelines_events.tsx index 52e3f6c48ea8f..94d01bd7162ed 100644 --- a/x-pack/plugins/security_solution/public/common/components/events_viewer/use_timelines_events.tsx +++ b/x-pack/plugins/security_solution/public/common/components/events_viewer/use_timelines_events.tsx @@ -27,8 +27,8 @@ import type { TimelineRequestSortField, TimelineStrategyResponseType, } from '@kbn/timelines-plugin/common/search_strategy'; -import type { MappingRuntimeFields } from '@elastic/elasticsearch/lib/api/types'; import { dataTableActions, Direction, TableId } from '@kbn/securitysolution-data-table'; +import type { RunTimeMappings } from '../../store/sourcerer/model'; import { TimelineEventsQueries } from '../../../../common/search_strategy'; import type { KueryFilterQueryKind } from '../../../../common/types'; import type { ESQuery } from '../../../../common/typed_json'; @@ -77,7 +77,7 @@ export interface UseTimelineEventsProps { indexNames: string[]; language?: KueryFilterQueryKind; limit: number; - runtimeMappings: MappingRuntimeFields; + runtimeMappings: RunTimeMappings; skip?: boolean; sort?: TimelineRequestSortField[]; startDate: string; @@ -321,7 +321,7 @@ export const useTimelineEventsHandler = ({ querySize: prevRequest?.pagination.querySize ?? 0, sort: prevRequest?.sort ?? initSortDefault, timerange: prevRequest?.timerange ?? {}, - runtimeMappings: prevRequest?.runtimeMappings ?? {}, + runtimeMappings: (prevRequest?.runtimeMappings ?? {}) as RunTimeMappings, filterStatus: prevRequest?.filterStatus, }; diff --git a/x-pack/plugins/security_solution/public/common/components/markdown_editor/plugins/insight/index.tsx b/x-pack/plugins/security_solution/public/common/components/markdown_editor/plugins/insight/index.tsx index e70e06e84c1f1..45b7e601267cd 100644 --- a/x-pack/plugins/security_solution/public/common/components/markdown_editor/plugins/insight/index.tsx +++ b/x-pack/plugins/security_solution/public/common/components/markdown_editor/plugins/insight/index.tsx @@ -32,6 +32,7 @@ import { import numeral from '@elastic/numeral'; import { css } from '@emotion/react'; import type { EuiMarkdownEditorUiPluginEditorProps } from '@elastic/eui/src/components/markdown_editor/markdown_types'; +import { DataView } from '@kbn/data-views-plugin/public'; import { FormattedMessage } from '@kbn/i18n-react'; import type { Filter } from '@kbn/es-query'; import { FilterStateStore } from '@kbn/es-query'; @@ -245,7 +246,16 @@ const InsightEditorComponent = ({ ui: { FiltersBuilderLazy }, }, uiSettings, + fieldFormats, } = useKibana().services; + const dataView = useMemo(() => { + if (sourcererDataView != null) { + return new DataView({ spec: sourcererDataView, fieldFormats }); + } else { + return null; + } + }, [sourcererDataView, fieldFormats]); + const [providers, setProviders] = useState([[]]); const dateRangeChoices = useMemo(() => { const settings: Array<{ from: string; to: string; display: string }> = uiSettings.get( @@ -419,11 +429,11 @@ const InsightEditorComponent = ({ /> - {sourcererDataView ? ( + {dataView ? ( ) : ( diff --git a/x-pack/plugins/security_solution/public/common/containers/source/__snapshots__/index.test.tsx.snap b/x-pack/plugins/security_solution/public/common/containers/source/__snapshots__/index.test.tsx.snap deleted file mode 100644 index df6ed6f80ed72..0000000000000 --- a/x-pack/plugins/security_solution/public/common/containers/source/__snapshots__/index.test.tsx.snap +++ /dev/null @@ -1,93 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`source/index.tsx getAllBrowserFields it returns an array of all fields in the BrowserFields argument 1`] = ` -Array [ - Object { - "aggregatable": true, - "count": 10, - "esTypes": Array [ - "long", - ], - "isMapped": true, - "name": "bytes", - "readFromDocValues": true, - "scripted": false, - "searchable": true, - "type": "number", - }, - Object { - "aggregatable": true, - "count": 20, - "esTypes": Array [ - "boolean", - ], - "isMapped": true, - "name": "ssl", - "readFromDocValues": true, - "scripted": false, - "searchable": true, - "type": "boolean", - }, - Object { - "aggregatable": true, - "count": 30, - "esTypes": Array [ - "date", - ], - "isMapped": true, - "name": "@timestamp", - "readFromDocValues": true, - "scripted": false, - "searchable": true, - "type": "date", - }, -] -`; - -exports[`source/index.tsx getBrowserFields it transforms input into output as expected 1`] = ` -Object { - "base": Object { - "fields": Object { - "@timestamp": Object { - "aggregatable": true, - "count": 30, - "esTypes": Array [ - "date", - ], - "isMapped": true, - "name": "@timestamp", - "readFromDocValues": true, - "scripted": false, - "searchable": true, - "type": "date", - }, - "bytes": Object { - "aggregatable": true, - "count": 10, - "esTypes": Array [ - "long", - ], - "isMapped": true, - "name": "bytes", - "readFromDocValues": true, - "scripted": false, - "searchable": true, - "type": "number", - }, - "ssl": Object { - "aggregatable": true, - "count": 20, - "esTypes": Array [ - "boolean", - ], - "isMapped": true, - "name": "ssl", - "readFromDocValues": true, - "scripted": false, - "searchable": true, - "type": "boolean", - }, - }, - }, -} -`; diff --git a/x-pack/plugins/security_solution/public/common/containers/source/index.test.tsx b/x-pack/plugins/security_solution/public/common/containers/source/index.test.tsx index 0f0347c02dc33..84c94f81d026f 100644 --- a/x-pack/plugins/security_solution/public/common/containers/source/index.test.tsx +++ b/x-pack/plugins/security_solution/public/common/containers/source/index.test.tsx @@ -5,8 +5,6 @@ * 2.0. */ -import type { IndexField } from '../../../../common/search_strategy/index_fields'; -import { getBrowserFields, getAllBrowserFields } from '.'; import type { IndexFieldSearch } from './use_data_view'; import { useDataView } from './use_data_view'; import { mocksSource } from './mock'; @@ -27,32 +25,6 @@ jest.mock('../../lib/kibana'); jest.mock('../../lib/apm/use_track_http_request'); describe('source/index.tsx', () => { - describe('getAllBrowserFields', () => { - test('it returns an array of all fields in the BrowserFields argument', () => { - expect( - getAllBrowserFields(getBrowserFields('title 1', mocksSource.indexFields as IndexField[])) - ).toMatchSnapshot(); - }); - }); - describe('getBrowserFields', () => { - test('it returns an empty object given an empty array', () => { - const fields = getBrowserFields('title 1', []); - expect(fields).toEqual({}); - }); - - test('it returns the same input given the same title and same fields length', () => { - const oldFields = getBrowserFields('title 1', mocksSource.indexFields as IndexField[]); - const newFields = getBrowserFields('title 1', mocksSource.indexFields as IndexField[]); - // Since it is memoized it will return the same object instance - expect(newFields).toBe(oldFields); - }); - - test('it transforms input into output as expected', () => { - const fields = getBrowserFields('title 2', mocksSource.indexFields as IndexField[]); - expect(fields).toMatchSnapshot(); - }); - }); - describe('useDataView hook', () => { const mockSearchResponse = { ...mocksSource, @@ -74,8 +46,8 @@ describe('source/index.tsx', () => { data: { dataViews: { ...useKibana().services.data.dataViews, - get: async (dataViewId: string, displayErrors?: boolean, refreshFields = false) => - Promise.resolve({ + get: async (dataViewId: string, displayErrors?: boolean, refreshFields = false) => { + const dataViewMock = { id: dataViewId, matchedIndices: refreshFields ? ['hello', 'world', 'refreshed'] @@ -88,7 +60,12 @@ describe('source/index.tsx', () => { type: 'keyword', }, }), - }), + }; + return Promise.resolve({ + toSpec: () => dataViewMock, + ...dataViewMock, + }); + }, getFieldsForWildcard: async () => Promise.resolve(), }, search: { @@ -178,7 +155,6 @@ describe('source/index.tsx', () => { const { payload: { patternList: newPatternList }, } = mockDispatch.mock.calls[1][0]; - expect(patternList).not.toBe(newPatternList); expect(patternList).not.toContain('refreshed*'); expect(newPatternList).toContain('refreshed*'); diff --git a/x-pack/plugins/security_solution/public/common/containers/source/index.tsx b/x-pack/plugins/security_solution/public/common/containers/source/index.tsx index 0144edec62dc8..7f259cdf403e3 100644 --- a/x-pack/plugins/security_solution/public/common/containers/source/index.tsx +++ b/x-pack/plugins/security_solution/public/common/containers/source/index.tsx @@ -9,9 +9,9 @@ import { isEmpty, isEqual, keyBy, pick } from 'lodash/fp'; import memoizeOne from 'memoize-one'; import { useCallback, useEffect, useRef, useState } from 'react'; import type { DataViewBase } from '@kbn/es-query'; -import type { BrowserField, BrowserFields, IndexField } from '@kbn/timelines-plugin/common'; -import type { DataView, IIndexPatternFieldList } from '@kbn/data-views-plugin/common'; -import { getCategory } from '@kbn/triggers-actions-ui-plugin/public'; +import type { BrowserField, BrowserFields } from '@kbn/timelines-plugin/common'; +import type { IIndexPatternFieldList } from '@kbn/data-views-plugin/common'; +import type { DataViewSpec } from '@kbn/data-views-plugin/public'; import { useKibana } from '../../lib/kibana'; import * as i18n from './translations'; @@ -72,35 +72,6 @@ export const getIndexFields = memoizeOne( newArgs[2] === lastArgs[2] ); -/** - * HOT Code path where the fields can be 16087 in length or larger. This is - * VERY mutatious on purpose to improve the performance of the transform. - */ -export const getBrowserFields = memoizeOne( - (_title: string, fields: IndexField[]): BrowserFields => { - // Adds two dangerous casts to allow for mutations within this function - type DangerCastForMutation = Record; - type DangerCastForBrowserFieldsMutation = Record< - string, - Omit & { fields: Record } - >; - - // We mutate this instead of using lodash/set to keep this as fast as possible - return fields.reduce((accumulator, field) => { - const category = getCategory(field.name); - if (accumulator[category] == null) { - (accumulator as DangerCastForMutation)[category] = {}; - } - if (accumulator[category].fields == null) { - accumulator[category].fields = {}; - } - accumulator[category].fields[field.name] = field as unknown as BrowserField; - return accumulator; - }, {}); - }, - (newArgs, lastArgs) => newArgs[0] === lastArgs[0] && newArgs[1].length === lastArgs[1].length -); - const DEFAULT_BROWSER_FIELDS = {}; const DEFAULT_INDEX_PATTERNS = { fields: [], title: '' }; interface FetchIndexReturn { @@ -115,7 +86,7 @@ interface FetchIndexReturn { indexes: string[]; indexExists: boolean; indexPatterns: DataViewBase; - dataView: DataView | undefined; + dataView: DataViewSpec | undefined; } /** @@ -149,9 +120,10 @@ export const useFetchIndex = ( setState({ ...state, loading: true }); abortCtrl.current = new AbortController(); const dv = await data.dataViews.create({ title: iNames.join(','), allowNoIndex: true }); + const dataView = dv.toSpec(); const { browserFields } = getDataViewStateFromIndexFields( iNames, - dv.fields, + dataView.fields, includeUnmapped ); @@ -159,7 +131,7 @@ export const useFetchIndex = ( setState({ loading: false, - dataView: dv, + dataView, browserFields, indexes: dv.getIndexPattern().split(','), indexExists: dv.getIndexPattern().split(',').length > 0, diff --git a/x-pack/plugins/security_solution/public/common/containers/source/mock.ts b/x-pack/plugins/security_solution/public/common/containers/source/mock.ts index d5511cbf92d6e..f253e694aa81c 100644 --- a/x-pack/plugins/security_solution/public/common/containers/source/mock.ts +++ b/x-pack/plugins/security_solution/public/common/containers/source/mock.ts @@ -5,7 +5,7 @@ * 2.0. */ -import type { MappingRuntimeFields } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { MappingRuntimeFieldType } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { DEFAULT_INDEX_PATTERN } from '../../../../common/constants'; import type { BrowserFields } from '../../../../common/search_strategy/index_fields'; @@ -577,11 +577,13 @@ export const mockBrowserFields: BrowserFields = { }, }; -export const mockRuntimeMappings: MappingRuntimeFields = { +const runTimeType: MappingRuntimeFieldType = 'keyword' as const; + +export const mockRuntimeMappings = { '@a.runtime.field': { script: { source: 'emit("Radical dude: " + doc[\'host.name\'].value)', }, - type: 'keyword', + type: runTimeType, }, }; diff --git a/x-pack/plugins/security_solution/public/common/containers/source/use_data_view.tsx b/x-pack/plugins/security_solution/public/common/containers/source/use_data_view.tsx index 5803e3fc6cb04..46bbd92f21c79 100644 --- a/x-pack/plugins/security_solution/public/common/containers/source/use_data_view.tsx +++ b/x-pack/plugins/security_solution/public/common/containers/source/use_data_view.tsx @@ -9,9 +9,9 @@ import { useCallback, useRef } from 'react'; import type { Subscription } from 'rxjs'; import { useDispatch } from 'react-redux'; import memoizeOne from 'memoize-one'; -import type { BrowserField } from '@kbn/timelines-plugin/common'; +import type { BrowserField, BrowserFields } from '@kbn/timelines-plugin/common'; import { getCategory } from '@kbn/triggers-actions-ui-plugin/public'; -import type { DataViewFieldBase } from '@kbn/es-query'; +import type { DataViewSpec } from '@kbn/data-views-plugin/public'; import { useKibana } from '../../lib/kibana'; import { sourcererActions } from '../../store/sourcerer'; @@ -50,34 +50,31 @@ interface DataViewInfo { export const getDataViewStateFromIndexFields = memoizeOne( ( _title: string, - fields: DataViewFieldBase[], + fields: DataViewSpec['fields'], _includeUnmapped: boolean = false ): DataViewInfo => { // Adds two dangerous casts to allow for mutations within this function type DangerCastForMutation = Record; - - return fields.reduce( - (acc, field) => { - // mutate browserFields - const category = getCategory(field.name); - if (acc.browserFields[category] == null) { - (acc.browserFields as DangerCastForMutation)[category] = {}; + if (fields == null) { + return { browserFields: {} }; + } else { + const browserFields: BrowserFields = {}; + for (const [name, field] of Object.entries(fields)) { + const category = getCategory(name); + if (browserFields[category] == null) { + (browserFields as DangerCastForMutation)[category] = { fields: {} }; } - if (acc.browserFields[category].fields == null) { - acc.browserFields[category].fields = {}; + const categoryFields = browserFields[category].fields; + if (categoryFields) { + categoryFields[name] = field as BrowserField; } - acc.browserFields[category].fields[field.name] = field as unknown as BrowserField; - - return acc; - }, - { - browserFields: {}, } - ); + return { browserFields: browserFields as DangerCastForBrowserFieldsMutation }; + } }, (newArgs, lastArgs) => newArgs[0] === lastArgs[0] && - newArgs[1].length === lastArgs[1].length && + newArgs[1]?.length === lastArgs[1]?.length && newArgs[2] === lastArgs[2] ); @@ -89,7 +86,6 @@ export const useDataView = (): { const searchSubscription$ = useRef>({}); const dispatch = useDispatch(); const { addError } = useAppToasts(); - const setLoading = useCallback( ({ id, loading }: { id: string; loading: boolean }) => { dispatch(sourcererActions.setDataViewLoading({ id, loading })); @@ -113,7 +109,6 @@ export const useDataView = (): { setLoading({ id: dataViewId, loading: true }); const dataView = await getSourcererDataView(dataViewId, data.dataViews, cleanCache); - if (needToBeInit && scopeId && !skipScopeUpdate) { dispatch( sourcererActions.setSelectedDataView({ diff --git a/x-pack/plugins/security_solution/public/common/containers/sourcerer/get_sourcerer_data_view.ts b/x-pack/plugins/security_solution/public/common/containers/sourcerer/get_sourcerer_data_view.ts index d7579abc2d8aa..7f40819bfb41d 100644 --- a/x-pack/plugins/security_solution/public/common/containers/sourcerer/get_sourcerer_data_view.ts +++ b/x-pack/plugins/security_solution/public/common/containers/sourcerer/get_sourcerer_data_view.ts @@ -7,7 +7,7 @@ import type { DataViewsContract } from '@kbn/data-views-plugin/common'; import { ensurePatternFormat } from '../../../../common/utils/sourcerer'; -import type { SourcererDataView } from '../../store/sourcerer/model'; +import type { SourcererDataView, RunTimeMappings } from '../../store/sourcerer/model'; import { getDataViewStateFromIndexFields } from '../source/use_data_view'; export const getSourcererDataView = async ( @@ -15,8 +15,9 @@ export const getSourcererDataView = async ( dataViewsService: DataViewsContract, refreshFields = false ): Promise => { - const dataViewData = await dataViewsService.get(dataViewId, true, refreshFields); - const defaultPatternsList = ensurePatternFormat(dataViewData.getIndexPattern().split(',')); + const dataView = await dataViewsService.get(dataViewId, true, refreshFields); + const dataViewData = dataView.toSpec(); + const defaultPatternsList = ensurePatternFormat(dataView.getIndexPattern().split(',')); // typeguard used to assert that pattern is a string, otherwise // typescript expects patternList to be (string | null)[] @@ -45,15 +46,13 @@ export const getSourcererDataView = async ( return { loading: false, id: dataViewData.id ?? '', - title: dataViewData.getIndexPattern(), - indexFields: dataViewData.fields, + title: dataView.getIndexPattern(), + indexFields: dataView.fields, fields: dataViewData.fields, patternList, dataView: dataViewData, - browserFields: getDataViewStateFromIndexFields( - dataViewData.id ?? '', - dataViewData.fields != null ? dataViewData.fields : [] - ).browserFields, - runtimeMappings: dataViewData.getRuntimeMappings(), + browserFields: getDataViewStateFromIndexFields(dataViewData.id ?? '', dataViewData.fields) + .browserFields, + runtimeMappings: dataViewData.runtimeFieldMap as RunTimeMappings, }; }; diff --git a/x-pack/plugins/security_solution/public/common/containers/sourcerer/index.test.tsx b/x-pack/plugins/security_solution/public/common/containers/sourcerer/index.test.tsx index 6c367fec1111f..6b534d64c8f9f 100644 --- a/x-pack/plugins/security_solution/public/common/containers/sourcerer/index.test.tsx +++ b/x-pack/plugins/security_solution/public/common/containers/sourcerer/index.test.tsx @@ -245,7 +245,7 @@ describe('Sourcerer Hooks', () => { type: 'x-pack/security_solution/local/sourcerer/SET_SOURCERER_SCOPE_LOADING', payload: { loading: false }, }); - expect(mockDispatch).toHaveBeenCalledTimes(9); + expect(mockDispatch).toHaveBeenCalledTimes(7); expect(mockSearch).toHaveBeenCalledTimes(2); }); }); diff --git a/x-pack/plugins/security_solution/public/common/containers/sourcerer/index.tsx b/x-pack/plugins/security_solution/public/common/containers/sourcerer/index.tsx index b3c17e530f84a..f17fccf7f29ed 100644 --- a/x-pack/plugins/security_solution/public/common/containers/sourcerer/index.tsx +++ b/x-pack/plugins/security_solution/public/common/containers/sourcerer/index.tsx @@ -14,6 +14,7 @@ import type { SelectedDataView, SourcererDataView, SourcererUrlState, + RunTimeMappings, } from '../../store/sourcerer/model'; import { SourcererScopeName } from '../../store/sourcerer/model'; import { useUserInfo } from '../../../detections/components/user_info'; @@ -395,14 +396,14 @@ export const useSourcererDataView = ( () => ({ ...fetchIndexReturn, dataView: fetchIndexReturn.dataView, - runtimeMappings: fetchIndexReturn.dataView?.getRuntimeMappings() ?? {}, - title: fetchIndexReturn.dataView?.getIndexPattern() ?? '', + runtimeMappings: (fetchIndexReturn.dataView?.runtimeFieldMap as RunTimeMappings) ?? {}, + title: fetchIndexReturn.dataView?.title ?? '', id: fetchIndexReturn.dataView?.id ?? null, loading: indexPatternsLoading, patternList: fetchIndexReturn.indexes, indexFields: fetchIndexReturn.indexPatterns .fields as SelectedDataView['indexPattern']['fields'], - fields: fetchIndexReturn.indexPatterns.fields, + fields: fetchIndexReturn.dataView?.fields, }), [fetchIndexReturn, indexPatternsLoading] ); diff --git a/x-pack/plugins/security_solution/public/common/mock/global_state.ts b/x-pack/plugins/security_solution/public/common/mock/global_state.ts index 343ddf6755ac9..8fdb43e14c96d 100644 --- a/x-pack/plugins/security_solution/public/common/mock/global_state.ts +++ b/x-pack/plugins/security_solution/public/common/mock/global_state.ts @@ -6,6 +6,7 @@ */ import { TableId } from '@kbn/securitysolution-data-table'; +import type { DataViewSpec } from '@kbn/data-views-plugin/public'; import { InputsModelId } from '../store/inputs/constants'; import { Direction, @@ -48,6 +49,10 @@ import { UsersFields } from '../../../common/search_strategy/security_solution/u import { initialGroupingState } from '../store/grouping/reducer'; import type { SourcererState } from '../store/sourcerer'; +const mockFieldMap: DataViewSpec['fields'] = Object.fromEntries( + mockIndexFields.map((field) => [field.name, field]) +); + export const mockSourcererState: SourcererState = { ...initialSourcererState, signalIndexName: `${DEFAULT_SIGNALS_INDEX}-spacename`, @@ -56,7 +61,7 @@ export const mockSourcererState: SourcererState = { browserFields: mockBrowserFields, id: DEFAULT_DATA_VIEW_ID, indexFields: mockIndexFields, - fields: mockIndexFields, + fields: mockFieldMap, loading: false, patternList: [...DEFAULT_INDEX_PATTERN, `${DEFAULT_SIGNALS_INDEX}-spacename`], runtimeMappings: mockRuntimeMappings, diff --git a/x-pack/plugins/security_solution/public/common/store/sourcerer/model.ts b/x-pack/plugins/security_solution/public/common/store/sourcerer/model.ts index 348a290dc59dc..d0569c24a5029 100644 --- a/x-pack/plugins/security_solution/public/common/store/sourcerer/model.ts +++ b/x-pack/plugins/security_solution/public/common/store/sourcerer/model.ts @@ -5,12 +5,12 @@ * 2.0. */ -import type { MappingRuntimeFields } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import type { DataView } from '@kbn/data-views-plugin/common'; -import type { DataViewFieldBase } from '@kbn/es-query'; import type { BrowserFields } from '@kbn/timelines-plugin/common'; import { EMPTY_BROWSER_FIELDS, EMPTY_INDEX_FIELDS } from '@kbn/timelines-plugin/common'; +import type { DataViewSpec } from '@kbn/data-views-plugin/public'; +import type { RuntimeFieldSpec, RuntimePrimitiveTypes } from '@kbn/data-views-plugin/common'; import type { SecuritySolutionDataViewBase } from '../../types'; + /** Uniquely identifies a Sourcerer Scope */ export enum SourcererScopeName { default = 'default', @@ -52,6 +52,10 @@ export interface KibanaDataView { title: string; } +export type RunTimeMappings = + | Record & { type: RuntimePrimitiveTypes }> + | undefined; + /** * DataView from Kibana + timelines/index_fields enhanced field data */ @@ -68,7 +72,7 @@ export interface SourcererDataView extends KibanaDataView { * @deprecated use sourcererDataView.fields * comes from dataView.fields.toSpec() */ indexFields: SecuritySolutionDataViewBase['fields']; - fields: DataViewFieldBase[]; + fields: DataViewSpec['fields'] | undefined; /** set when data view fields are fetched */ loading: boolean; /** @@ -76,11 +80,11 @@ export interface SourcererDataView extends KibanaDataView { * Needed to pass to search strategy * Remove once issue resolved: https://github.com/elastic/kibana/issues/111762 */ - runtimeMappings: MappingRuntimeFields; + runtimeMappings: RunTimeMappings; /** * @type DataView @kbn/data-views-plugin/common */ - dataView: DataView | undefined; + dataView: DataViewSpec | undefined; } /** @@ -125,7 +129,7 @@ export interface SelectedDataView { * Easier to add this additional data rather than * try to extend the SelectedDataView type from DataView. */ - sourcererDataView: DataView | undefined; + sourcererDataView: DataViewSpec | undefined; } /** @@ -155,11 +159,12 @@ export const initSourcererScope: Omit = { selectedPatterns: [], missingPatterns: [], }; + export const initDataView: SourcererDataView & { id: string; error?: unknown } = { browserFields: EMPTY_BROWSER_FIELDS, id: '', indexFields: EMPTY_INDEX_FIELDS, - fields: EMPTY_INDEX_FIELDS, + fields: undefined, loading: false, patternList: [], runtimeMappings: {}, diff --git a/x-pack/plugins/security_solution/public/detections/components/alerts_table/alerts_grouping.tsx b/x-pack/plugins/security_solution/public/detections/components/alerts_table/alerts_grouping.tsx index b05d456b3be26..6464dbbad93ce 100644 --- a/x-pack/plugins/security_solution/public/detections/components/alerts_table/alerts_grouping.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/alerts_table/alerts_grouping.tsx @@ -6,7 +6,6 @@ */ import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'; -import type { MappingRuntimeFields } from '@elastic/elasticsearch/lib/api/types'; import { useDispatch, useSelector } from 'react-redux'; import type { Filter, Query } from '@kbn/es-query'; import type { GroupOption } from '@kbn/securitysolution-grouping'; @@ -21,6 +20,7 @@ import type { Status } from '../../../../common/detection_engine/schemas/common' import { defaultUnit } from '../../../common/components/toolbar/unit'; import { useSourcererDataView } from '../../../common/containers/sourcerer'; import { SourcererScopeName } from '../../../common/store/sourcerer/model'; +import type { RunTimeMappings } from '../../../common/store/sourcerer/model'; import { getDefaultGroupingOptions, renderGroupPanel, getStats } from './grouping_settings'; import { useKibana } from '../../../common/lib/kibana'; import { GroupedSubLevel } from './alerts_sub_grouping'; @@ -36,7 +36,7 @@ export interface AlertsTableComponentProps { hasIndexWrite: boolean; loading: boolean; renderChildComponent: (groupingFilters: Filter[]) => React.ReactElement; - runtimeMappings: MappingRuntimeFields; + runtimeMappings: RunTimeMappings; signalIndexName: string | null; tableId: TableIdLiteral; to: string; diff --git a/x-pack/plugins/security_solution/public/detections/components/alerts_table/alerts_sub_grouping.tsx b/x-pack/plugins/security_solution/public/detections/components/alerts_table/alerts_sub_grouping.tsx index c740df7e273f2..f2fa9e8bb6d1b 100644 --- a/x-pack/plugins/security_solution/public/detections/components/alerts_table/alerts_sub_grouping.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/alerts_table/alerts_sub_grouping.tsx @@ -13,9 +13,9 @@ import type { GroupingAggregation } from '@kbn/securitysolution-grouping'; import { isNoneGroup } from '@kbn/securitysolution-grouping'; import { getEsQueryConfig } from '@kbn/data-plugin/common'; import type { DynamicGroupingProps } from '@kbn/securitysolution-grouping/src'; -import type { MappingRuntimeFields } from '@elastic/elasticsearch/lib/api/types'; import type { TableIdLiteral } from '@kbn/securitysolution-data-table'; import { parseGroupingQuery } from '@kbn/securitysolution-grouping/src'; +import type { RunTimeMappings } from '../../../common/store/sourcerer/model'; import { combineQueries } from '../../../common/lib/kuery'; import { SourcererScopeName } from '../../../common/store/sourcerer/model'; import type { AlertsGroupingAggregation } from './grouping_settings/types'; @@ -53,7 +53,7 @@ interface OwnProps { pageSize: number; parentGroupingFilter?: string; renderChildComponent: (groupingFilters: Filter[]) => React.ReactElement; - runtimeMappings: MappingRuntimeFields; + runtimeMappings: RunTimeMappings; selectedGroup: string; setPageIndex: (newIndex: number) => void; setPageSize: (newSize: number) => void; diff --git a/x-pack/plugins/security_solution/public/detections/components/alerts_table/grouping_settings/query_builder.ts b/x-pack/plugins/security_solution/public/detections/components/alerts_table/grouping_settings/query_builder.ts index 4da4f4fecfbcf..eda89f3d537ef 100644 --- a/x-pack/plugins/security_solution/public/detections/components/alerts_table/grouping_settings/query_builder.ts +++ b/x-pack/plugins/security_solution/public/detections/components/alerts_table/grouping_settings/query_builder.ts @@ -5,10 +5,10 @@ * 2.0. */ -import type { MappingRuntimeFields } from '@elastic/elasticsearch/lib/api/types'; import type { BoolQuery } from '@kbn/es-query'; import type { NamedAggregation } from '@kbn/securitysolution-grouping'; import { isNoneGroup, getGroupingQuery } from '@kbn/securitysolution-grouping'; +import type { RunTimeMappings } from '../../../../common/store/sourcerer/model'; interface AlertsGroupingQueryParams { additionalFilters: Array<{ @@ -17,7 +17,7 @@ interface AlertsGroupingQueryParams { from: string; pageIndex: number; pageSize: number; - runtimeMappings: MappingRuntimeFields; + runtimeMappings: RunTimeMappings; selectedGroup: string; uniqueValue: string; to: string; diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/chart_panels/index.tsx b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/chart_panels/index.tsx index 8d0fd71dbc0c3..8f600e3b54f71 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/chart_panels/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/chart_panels/index.tsx @@ -5,7 +5,6 @@ * 2.0. */ -import type { MappingRuntimeFields } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { Filter, Query } from '@kbn/es-query'; import { EuiFlexItem, EuiLoadingContent } from '@elastic/eui'; import React, { useCallback, useMemo } from 'react'; @@ -19,6 +18,7 @@ import { ChartSelect } from './chart_select'; import { ChartCollapse } from './chart_collapse'; import * as i18n from './chart_select/translations'; import { AlertsTreemapPanel } from '../../../../common/components/alerts_treemap_panel'; +import type { RunTimeMappings } from '../../../../common/store/sourcerer/model'; import type { UpdateDateRange } from '../../../../common/components/charts/common'; import { useEuiComboBoxReset } from '../../../../common/components/use_combo_box_reset'; import { AlertsHistogramPanel } from '../../../components/alerts_kpis/alerts_histogram_panel'; @@ -51,7 +51,7 @@ export interface Props { alertsDefaultFilters: Filter[]; isLoadingIndexPattern: boolean; query: Query; - runtimeMappings: MappingRuntimeFields; + runtimeMappings: RunTimeMappings; signalIndexName: string | null; updateDateRangeCallback: UpdateDateRange; } diff --git a/x-pack/plugins/security_solution/public/timelines/components/side_panel/event_details/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/side_panel/event_details/index.tsx index 059f4322d8970..957bd82e675c1 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/side_panel/event_details/index.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/side_panel/event_details/index.tsx @@ -9,12 +9,12 @@ import { EuiSpacer, EuiFlyoutBody } from '@elastic/eui'; import React, { useMemo } from 'react'; import deepEqual from 'fast-deep-equal'; -import type { MappingRuntimeFields } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { EntityType } from '@kbn/timelines-plugin/common'; import type { BrowserFields } from '../../../../common/containers/source'; import { ExpandableEvent, ExpandableEventTitle } from './expandable_event'; import { useTimelineEventsDetails } from '../../../containers/details'; import type { TimelineTabs } from '../../../../../common/types/timeline'; +import type { RunTimeMappings } from '../../../../common/store/sourcerer/model'; import { useHostIsolationTools } from './use_host_isolation_tools'; import { FlyoutBody, FlyoutHeader, FlyoutFooter } from './flyout'; import { useBasicDataFromDetailsData, getAlertIndexAlias } from './helpers'; @@ -33,7 +33,7 @@ interface EventDetailsPanelProps { handleOnEventClosed: () => void; isDraggable?: boolean; isFlyoutView?: boolean; - runtimeMappings: MappingRuntimeFields; + runtimeMappings: RunTimeMappings; tabType: TimelineTabs; scopeId: string; isReadOnly?: boolean; diff --git a/x-pack/plugins/security_solution/public/timelines/components/side_panel/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/side_panel/index.tsx index c92c49cda23d6..3831188540781 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/side_panel/index.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/side_panel/index.tsx @@ -10,13 +10,13 @@ import { useDispatch } from 'react-redux'; import type { EuiFlyoutProps } from '@elastic/eui'; import { EuiFlyout } from '@elastic/eui'; -import type { MappingRuntimeFields } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { EntityType } from '@kbn/timelines-plugin/common'; import { dataTableActions, dataTableSelectors } from '@kbn/securitysolution-data-table'; import { getScopedActions, isInTableScope, isTimelineScope } from '../../../helpers'; import { timelineSelectors } from '../../store/timeline'; import { timelineDefaults } from '../../store/timeline/defaults'; import type { BrowserFields } from '../../../common/containers/source'; +import type { RunTimeMappings } from '../../../common/store/sourcerer/model'; import { TimelineId, TimelineTabs } from '../../../../common/types/timeline'; import { useDeepEqualSelector } from '../../../common/hooks/use_selector'; import { EventDetailsPanel } from './event_details'; @@ -30,7 +30,7 @@ interface DetailsPanelProps { entityType?: EntityType; handleOnPanelClosed?: () => void; isFlyoutView?: boolean; - runtimeMappings: MappingRuntimeFields; + runtimeMappings: RunTimeMappings; tabType?: TimelineTabs; scopeId: string; isReadOnly?: boolean; diff --git a/x-pack/plugins/security_solution/public/timelines/containers/details/index.tsx b/x-pack/plugins/security_solution/public/timelines/containers/details/index.tsx index 13841859c37ff..c07e52b09874f 100644 --- a/x-pack/plugins/security_solution/public/timelines/containers/details/index.tsx +++ b/x-pack/plugins/security_solution/public/timelines/containers/details/index.tsx @@ -11,11 +11,11 @@ import ReactDOM from 'react-dom'; import deepEqual from 'fast-deep-equal'; import { Subscription } from 'rxjs'; -import type { MappingRuntimeFields } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { isCompleteResponse, isErrorResponse } from '@kbn/data-plugin/common'; import { EntityType } from '@kbn/timelines-plugin/common'; import type { EcsSecurityExtension as Ecs } from '@kbn/securitysolution-ecs'; import { useKibana } from '../../../common/lib/kibana'; +import type { RunTimeMappings } from '../../../common/store/sourcerer/model'; import type { SearchHit, TimelineEventsDetailsItem, @@ -35,7 +35,7 @@ export interface UseTimelineEventsDetailsProps { entityType?: EntityType; indexName: string; eventId: string; - runtimeMappings: MappingRuntimeFields; + runtimeMappings: RunTimeMappings; skip: boolean; } diff --git a/x-pack/plugins/security_solution/public/timelines/containers/index.tsx b/x-pack/plugins/security_solution/public/timelines/containers/index.tsx index 55c8d2cad65bc..0bcd156b6829c 100644 --- a/x-pack/plugins/security_solution/public/timelines/containers/index.tsx +++ b/x-pack/plugins/security_solution/public/timelines/containers/index.tsx @@ -11,12 +11,12 @@ import { useCallback, useEffect, useRef, useState } from 'react'; import { useDispatch } from 'react-redux'; import { Subscription } from 'rxjs'; -import type { MappingRuntimeFields } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { DataView } from '@kbn/data-plugin/common'; import { isCompleteResponse, isErrorResponse } from '@kbn/data-plugin/common'; import type { ESQuery } from '../../../common/typed_json'; import type { inputsModel } from '../../common/store'; +import type { RunTimeMappings } from '../../common/store/sourcerer/model'; import { useKibana } from '../../common/lib/kibana'; import { createFilter } from '../../common/containers/helpers'; import { timelineActions } from '../store/timeline'; @@ -89,7 +89,7 @@ export interface UseTimelineEventsProps { indexNames: string[]; language?: KueryFilterQueryKind; limit: number; - runtimeMappings: MappingRuntimeFields; + runtimeMappings: RunTimeMappings; skip?: boolean; sort?: TimelineRequestSortField[]; startDate?: string; @@ -356,7 +356,7 @@ export const useTimelineEventsHandler = ({ querySize: prevRequest?.pagination.querySize ?? 0, sort: prevRequest?.sort ?? initSortDefault, timerange: prevRequest?.timerange ?? {}, - runtimeMappings: prevRequest?.runtimeMappings ?? {}, + runtimeMappings: (prevRequest?.runtimeMappings ?? {}) as RunTimeMappings, ...deStructureEqlOptions(prevEqlRequest), }; diff --git a/x-pack/plugins/security_solution/public/types.ts b/x-pack/plugins/security_solution/public/types.ts index e6e10a9c5b3dd..f4f46205a600e 100644 --- a/x-pack/plugins/security_solution/public/types.ts +++ b/x-pack/plugins/security_solution/public/types.ts @@ -8,6 +8,7 @@ import type { AppLeaveHandler, CoreStart } from '@kbn/core/public'; import type { HomePublicPluginSetup } from '@kbn/home-plugin/public'; import type { DataPublicPluginStart } from '@kbn/data-plugin/public'; +import type { FieldFormatsStartCommon } from '@kbn/field-formats-plugin/common'; import type { EmbeddableStart } from '@kbn/embeddable-plugin/public'; import type { LensPublicStart } from '@kbn/lens-plugin/public'; import type { NewsfeedPublicPluginStart } from '@kbn/newsfeed-plugin/public'; @@ -104,6 +105,7 @@ export interface StartPlugins { threatIntelligence: ThreatIntelligencePluginStart; cloudExperiments?: CloudExperimentsPluginStart; dataViews: DataViewsServicePublic; + fieldFormats: FieldFormatsStartCommon; } export interface StartPluginsDependencies extends StartPlugins { diff --git a/x-pack/plugins/security_solution/tsconfig.json b/x-pack/plugins/security_solution/tsconfig.json index 748130f6be7b1..39f607e136b5e 100644 --- a/x-pack/plugins/security_solution/tsconfig.json +++ b/x-pack/plugins/security_solution/tsconfig.json @@ -156,5 +156,6 @@ "@kbn/core-lifecycle-browser", "@kbn/ecs", "@kbn/url-state", + "@kbn/field-formats-plugin", ] } diff --git a/x-pack/plugins/timelines/common/search_strategy/index_fields/index.ts b/x-pack/plugins/timelines/common/search_strategy/index_fields/index.ts index 5e9b70536ac4c..70591424ff560 100644 --- a/x-pack/plugins/timelines/common/search_strategy/index_fields/index.ts +++ b/x-pack/plugins/timelines/common/search_strategy/index_fields/index.ts @@ -78,7 +78,7 @@ export interface BrowserField { category: string; description: string | null; example: string | number | null; - fields: Readonly>>; + fields: Record>; format: string; indexes: string[]; name: string; @@ -97,7 +97,7 @@ export interface BrowserField { * you are working with? Or perhaps you need a description for a * particular field? Consider using the EcsFlat module from `@kbn/ecs` */ -export type BrowserFields = Readonly>>; +export type BrowserFields = Record>; export const EMPTY_BROWSER_FIELDS = {}; export const EMPTY_INDEX_FIELDS: FieldSpec[] = []; diff --git a/x-pack/plugins/timelines/common/search_strategy/timeline/events/all/index.ts b/x-pack/plugins/timelines/common/search_strategy/timeline/events/all/index.ts index ff8a61602239c..ce2094334889f 100644 --- a/x-pack/plugins/timelines/common/search_strategy/timeline/events/all/index.ts +++ b/x-pack/plugins/timelines/common/search_strategy/timeline/events/all/index.ts @@ -7,11 +7,11 @@ import { JsonObject } from '@kbn/utility-types'; -import { MappingRuntimeFields } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { IEsSearchResponse } from '@kbn/data-plugin/common'; import type { EcsSecurityExtension as Ecs } from '@kbn/securitysolution-ecs'; import type { CursorType, Inspect, Maybe, PaginationInputPaginated } from '../../../common'; import type { TimelineRequestOptionsPaginated } from '../..'; +import type { RunTimeMappings } from '../eql'; export interface TimelineEdges { node: TimelineItem; @@ -45,6 +45,6 @@ export interface TimelineEventsAllRequestOptions extends TimelineRequestOptionsP fieldRequested: string[]; fields: string[] | Array<{ field: string; include_unmapped: boolean }>; language: 'eql' | 'kuery' | 'lucene'; - runtimeMappings: MappingRuntimeFields; + runtimeMappings: RunTimeMappings; filterStatus?: AlertWorkflowStatus; } diff --git a/x-pack/plugins/timelines/common/search_strategy/timeline/events/eql/index.ts b/x-pack/plugins/timelines/common/search_strategy/timeline/events/eql/index.ts index dac5d5651003c..4c9419ccf802a 100644 --- a/x-pack/plugins/timelines/common/search_strategy/timeline/events/eql/index.ts +++ b/x-pack/plugins/timelines/common/search_strategy/timeline/events/eql/index.ts @@ -6,10 +6,20 @@ */ import { EuiComboBoxOptionOption } from '@elastic/eui'; -import type { EqlSearchStrategyRequest, EqlSearchStrategyResponse } from '@kbn/data-plugin/common'; +import type { + EqlSearchStrategyRequest, + EqlSearchStrategyResponse, + EqlRequestParams, +} from '@kbn/data-plugin/common'; +import type { RuntimeFieldSpec, RuntimePrimitiveTypes } from '@kbn/data-views-plugin/common'; import { EqlSearchResponse, Inspect, Maybe, PaginationInputPaginated } from '../../..'; import { TimelineEdges, TimelineEventsAllRequestOptions } from '../..'; +type EqlBody = Pick; + +export type RunTimeMappings = + | Record & { type: RuntimePrimitiveTypes }> + | undefined; export interface TimelineEqlRequestOptions extends EqlSearchStrategyRequest, Omit { @@ -17,6 +27,8 @@ export interface TimelineEqlRequestOptions tiebreakerField?: string; timestampField?: string; size?: number; + runtime_mappings?: RunTimeMappings; + body?: Omit & EqlBody & { runtime_mappings?: RunTimeMappings }; } export interface TimelineEqlResponse extends EqlSearchStrategyResponse> { diff --git a/x-pack/plugins/timelines/common/search_strategy/timeline/index.ts b/x-pack/plugins/timelines/common/search_strategy/timeline/index.ts index a5856169a5748..608cc49f53120 100644 --- a/x-pack/plugins/timelines/common/search_strategy/timeline/index.ts +++ b/x-pack/plugins/timelines/common/search_strategy/timeline/index.ts @@ -5,7 +5,6 @@ * 2.0. */ -import type { MappingRuntimeFields } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { IEsSearchRequest } from '@kbn/data-plugin/common'; import { ESQuery } from '../../typed_json'; import { @@ -26,6 +25,7 @@ import { TimelineStatus, RowRendererId, } from '../../types/timeline'; +import type { RunTimeMappings } from './events/eql'; export * from './events'; @@ -37,7 +37,7 @@ export interface TimelineRequestBasicOptions extends IEsSearchRequest { defaultIndex: string[]; factoryQueryType?: TimelineFactoryQueryTypes; entityType?: EntityType; - runtimeMappings: MappingRuntimeFields; + runtimeMappings: RunTimeMappings; } export interface TimelineRequestSortField extends SortField {