From de38d64b3762a09ff1f3e3919700d351d5010576 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Sun, 30 Aug 2020 09:39:25 +0200 Subject: [PATCH] [Security Solution] Refactor NetworkTopNFlow to use Search Strategy --- .../security_solution/index.ts | 11 + .../security_solution/network/common.ts | 41 +++ .../security_solution/network/http.ts | 71 +++++ .../security_solution/network/index.ts | 181 +---------- .../network/top_countries.ts | 69 ++++ .../security_solution/network/top_n_flow.ts | 124 ++++++++ .../containers/network_top_n_flow/index.tsx | 296 ++++++++++-------- .../network_top_n_flow/translations.ts | 21 ++ .../network_top_n_flow_query_table.tsx | 74 ++--- .../pages/navigation/ips_query_tab_body.tsx | 72 ++--- .../factory/network/helpers.ts | 32 +- .../factory/network/index.ts | 2 + .../factory/network/top_n_flow/helpers.ts | 97 ++++++ .../factory/network/top_n_flow/index.ts | 59 ++++ .../query.top_n_flow_network.dsl.ts | 155 +++++++++ 15 files changed, 922 insertions(+), 383 deletions(-) create mode 100644 x-pack/plugins/security_solution/common/search_strategy/security_solution/network/common.ts create mode 100644 x-pack/plugins/security_solution/common/search_strategy/security_solution/network/http.ts create mode 100644 x-pack/plugins/security_solution/common/search_strategy/security_solution/network/top_countries.ts create mode 100644 x-pack/plugins/security_solution/common/search_strategy/security_solution/network/top_n_flow.ts create mode 100644 x-pack/plugins/security_solution/public/network/containers/network_top_n_flow/translations.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/helpers.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/index.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/query.top_n_flow_network.dsl.ts diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/index.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/index.ts index df9099ecabd38..ca444e8b6e12f 100644 --- a/x-pack/plugins/security_solution/common/search_strategy/security_solution/index.ts +++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/index.ts @@ -21,6 +21,8 @@ import { NetworkHttpRequestOptions, NetworkTopCountriesStrategyResponse, NetworkTopCountriesRequestOptions, + NetworkTopNFlowStrategyResponse, + NetworkTopNFlowRequestOptions, } from './network'; export * from './hosts'; @@ -117,6 +119,8 @@ export type StrategyResponseType = T extends HostsQ ? NetworkHttpStrategyResponse : T extends NetworkQueries.topCountries ? NetworkTopCountriesStrategyResponse + : T extends NetworkQueries.topNFlow + ? NetworkTopNFlowStrategyResponse : never; export type StrategyRequestType = T extends HostsQueries.hosts @@ -129,9 +133,16 @@ export type StrategyRequestType = T extends HostsQu ? NetworkHttpRequestOptions : T extends NetworkQueries.topCountries ? NetworkTopCountriesRequestOptions + : T extends NetworkQueries.topNFlow + ? NetworkTopNFlowRequestOptions : never; export interface GenericBuckets { key: string; doc_count: number; } + +export interface TotalValue { + value: number; + relation: string; +} diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/common.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/common.ts new file mode 100644 index 0000000000000..043be03972ed6 --- /dev/null +++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/common.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { GeoEcs } from '../../../ecs/geo'; +import { Maybe } from '..'; + +export enum NetworkTopTablesFields { + bytes_in = 'bytes_in', + bytes_out = 'bytes_out', + flows = 'flows', + destination_ips = 'destination_ips', + source_ips = 'source_ips', +} + +export interface NetworkTopTablesSortField { + field: NetworkTopTablesFields; + direction: Direction; +} + +export enum FlowTargetSourceDest { + destination = 'destination', + source = 'source', +} + +export enum Direction { + asc = 'asc', + desc = 'desc', +} + +export interface GeoItem { + geo?: Maybe; + flowTarget?: Maybe; +} + +export interface TopNetworkTablesEcsField { + bytes_in?: Maybe; + bytes_out?: Maybe; +} diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/http.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/http.ts new file mode 100644 index 0000000000000..568444eaa36de --- /dev/null +++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/http.ts @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IEsSearchResponse } from '../../../../../../../src/plugins/data/common'; +import { Direction } from './common'; +import { + CursorType, + GenericBuckets, + Inspect, + Maybe, + PageInfoPaginated, + RequestOptionsPaginated, +} from '..'; + +export interface NetworkHttpRequestOptions extends RequestOptionsPaginated { + ip?: string; + networkHttpSort: NetworkHttpSortField; + defaultIndex: string[]; +} + +export interface NetworkHttpStrategyResponse extends IEsSearchResponse { + edges: NetworkHttpEdges[]; + totalCount: number; + pageInfo: PageInfoPaginated; + inspect?: Maybe; +} + +export interface NetworkHttpSortField { + direction: Direction; +} + +export interface NetworkHttpData { + edges: NetworkHttpEdges[]; + totalCount: number; + pageInfo: PageInfoPaginated; + inspect?: Maybe; +} + +export interface NetworkHttpEdges { + node: NetworkHttpItem; + cursor: CursorType; +} + +export interface NetworkHttpItem { + _id?: Maybe; + domains: string[]; + lastHost?: Maybe; + lastSourceIp?: Maybe; + methods: string[]; + path?: Maybe; + requestCount?: Maybe; + statuses: string[]; +} + +export interface NetworkHttpBuckets { + key: string; + doc_count: number; + domains: { + buckets: GenericBuckets[]; + }; + methods: { + buckets: GenericBuckets[]; + }; + source: object; + status: { + buckets: GenericBuckets[]; + }; +} diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/index.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/index.ts index 347f3c56749d7..bfb8f31f4b300 100644 --- a/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/index.ts +++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/index.ts @@ -5,29 +5,19 @@ */ import { IEsSearchResponse } from '../../../../../../../src/plugins/data/common'; -import { GeoEcs } from '../../../ecs/geo'; -import { - CursorType, - Inspect, - Maybe, - PageInfoPaginated, - RequestOptionsPaginated, - GenericBuckets, -} from '..'; +import { CursorType, Inspect, Maybe, PageInfoPaginated, RequestOptionsPaginated } from '..'; + +export * from './common'; +export * from './http'; +export * from './top_countries'; +export * from './top_n_flow'; export enum NetworkQueries { http = 'http', tls = 'tls', topCountries = 'topCountries', -} - -export enum NetworkTopTablesFields { - bytes_in = 'bytes_in', - bytes_out = 'bytes_out', - flows = 'flows', - destination_ips = 'destination_ips', - source_ips = 'source_ips', + topNFlow = 'topNFlow', } export enum NetworkDnsFields { @@ -109,10 +99,6 @@ export interface TlsSortField { direction: Direction; } -export interface NetworkHttpSortField { - direction: Direction; -} - export interface NetworkTlsRequestOptions extends RequestOptionsPaginated { ip: string; flowTarget: FlowTargetSourceDest; @@ -129,156 +115,3 @@ export interface NetworkTlsStrategyResponse extends IEsSearchResponse { inspect?: Maybe; } - -export interface NetworkHttpRequestOptions extends RequestOptionsPaginated { - ip?: string; - networkHttpSort: NetworkHttpSortField; - defaultIndex: string[]; -} - -export interface NetworkHttpStrategyResponse extends IEsSearchResponse { - edges: NetworkHttpEdges[]; - - totalCount: number; - - pageInfo: PageInfoPaginated; - - inspect?: Maybe; -} - -export interface GeoItem { - geo?: Maybe; - - flowTarget?: Maybe; -} - -export interface TopCountriesItemSource { - country?: Maybe; - - destination_ips?: Maybe; - - flows?: Maybe; - - location?: Maybe; - - source_ips?: Maybe; -} - -export interface NetworkTopCountriesRequestOptions extends RequestOptionsPaginated { - networkTopCountriesSort: NetworkTopTablesSortField; - flowTarget: FlowTargetSourceDest; - ip?: string; -} - -export interface NetworkTopCountriesStrategyResponse extends IEsSearchResponse { - edges: NetworkTopCountriesEdges[]; - - totalCount: number; - - pageInfo: PageInfoPaginated; - - inspect?: Maybe; -} - -export interface NetworkTopCountriesEdges { - node: NetworkTopCountriesItem; - - cursor: CursorType; -} - -export interface NetworkTopCountriesItem { - _id?: Maybe; - - source?: Maybe; - - destination?: Maybe; - - network?: Maybe; -} - -export interface TopCountriesItemDestination { - country?: Maybe; - - destination_ips?: Maybe; - - flows?: Maybe; - - location?: Maybe; - - source_ips?: Maybe; -} - -export interface TopNetworkTablesEcsField { - bytes_in?: Maybe; - - bytes_out?: Maybe; -} - -export interface NetworkTopTablesSortField { - field: NetworkTopTablesFields; - - direction: Direction; -} - -export interface NetworkTopCountriesBuckets { - country: string; - key: string; - bytes_in: { - value: number; - }; - bytes_out: { - value: number; - }; - flows: number; - destination_ips: number; - source_ips: number; -} - -export interface NetworkHttpData { - edges: NetworkHttpEdges[]; - - totalCount: number; - - pageInfo: PageInfoPaginated; - - inspect?: Maybe; -} - -export interface NetworkHttpEdges { - node: NetworkHttpItem; - - cursor: CursorType; -} - -export interface NetworkHttpItem { - _id?: Maybe; - - domains: string[]; - - lastHost?: Maybe; - - lastSourceIp?: Maybe; - - methods: string[]; - - path?: Maybe; - - requestCount?: Maybe; - - statuses: string[]; -} - -export interface NetworkHttpBuckets { - key: string; - doc_count: number; - domains: { - buckets: GenericBuckets[]; - }; - methods: { - buckets: GenericBuckets[]; - }; - source: object; - status: { - buckets: GenericBuckets[]; - }; -} diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/top_countries.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/top_countries.ts new file mode 100644 index 0000000000000..9d1895447ecff --- /dev/null +++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/top_countries.ts @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IEsSearchResponse } from '../../../../../../../src/plugins/data/common'; +import { + GeoItem, + NetworkTopTablesSortField, + FlowTargetSourceDest, + TopNetworkTablesEcsField, +} from './common'; +import { CursorType, Inspect, Maybe, PageInfoPaginated, RequestOptionsPaginated } from '..'; + +export interface TopCountriesItemSource { + country?: Maybe; + destination_ips?: Maybe; + flows?: Maybe; + location?: Maybe; + source_ips?: Maybe; +} + +export interface NetworkTopCountriesRequestOptions extends RequestOptionsPaginated { + networkTopCountriesSort: NetworkTopTablesSortField; + flowTarget: FlowTargetSourceDest; + ip?: string; +} + +export interface NetworkTopCountriesStrategyResponse extends IEsSearchResponse { + edges: NetworkTopCountriesEdges[]; + totalCount: number; + pageInfo: PageInfoPaginated; + inspect?: Maybe; +} + +export interface NetworkTopCountriesEdges { + node: NetworkTopCountriesItem; + cursor: CursorType; +} + +export interface NetworkTopCountriesItem { + _id?: Maybe; + source?: Maybe; + destination?: Maybe; + network?: Maybe; +} + +export interface TopCountriesItemDestination { + country?: Maybe; + destination_ips?: Maybe; + flows?: Maybe; + location?: Maybe; + source_ips?: Maybe; +} + +export interface NetworkTopCountriesBuckets { + country: string; + key: string; + bytes_in: { + value: number; + }; + bytes_out: { + value: number; + }; + flows: number; + destination_ips: number; + source_ips: number; +} diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/top_n_flow.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/top_n_flow.ts new file mode 100644 index 0000000000000..4dbf6a47ae01f --- /dev/null +++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/top_n_flow.ts @@ -0,0 +1,124 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IEsSearchResponse } from '../../../../../../../src/plugins/data/common'; +import { + GeoItem, + NetworkTopTablesSortField, + FlowTargetSourceDest, + TopNetworkTablesEcsField, +} from './common'; +import { + CursorType, + GenericBuckets, + Inspect, + Maybe, + PageInfoPaginated, + RequestOptionsPaginated, + TotalValue, +} from '..'; + +export interface NetworkTopNFlowRequestOptions extends RequestOptionsPaginated { + networkTopNFlowSort: NetworkTopTablesSortField; + flowTarget: FlowTargetSourceDest; + ip?: Maybe; +} + +export interface NetworkTopNFlowStrategyResponse extends IEsSearchResponse { + edges: NetworkTopNFlowEdges[]; + totalCount: number; + pageInfo: PageInfoPaginated; + inspect?: Maybe; +} + +export interface NetworkTopNFlowEdges { + node: NetworkTopNFlowItem; + cursor: CursorType; +} + +export interface NetworkTopNFlowItem { + _id?: Maybe; + source?: Maybe; + destination?: Maybe; + network?: Maybe; +} + +export interface TopNFlowItemSource { + autonomous_system?: Maybe; + domain?: Maybe; + ip?: Maybe; + location?: Maybe; + flows?: Maybe; + destination_ips?: Maybe; +} + +export interface AutonomousSystemItem { + name?: Maybe; + number?: Maybe; +} + +export interface TopNFlowItemDestination { + autonomous_system?: Maybe; + domain?: Maybe; + ip?: Maybe; + location?: Maybe; + flows?: Maybe; + source_ips?: Maybe; +} + +export interface AutonomousSystemHit { + doc_count: number; + top_as: { + hits: { + total: TotalValue | number; + max_score: number | null; + hits: Array<{ + _source: T; + sort?: [number]; + _index?: string; + _type?: string; + _id?: string; + _score?: number | null; + }>; + }; + }; +} + +export interface NetworkTopNFlowBuckets { + key: string; + autonomous_system: AutonomousSystemHit; + bytes_in: { + value: number; + }; + bytes_out: { + value: number; + }; + domain: { + buckets: GenericBuckets[]; + }; + location: LocationHit; + flows: number; + destination_ips?: number; + source_ips?: number; +} + +export interface LocationHit { + doc_count: number; + top_geo: { + hits: { + total: TotalValue | number; + max_score: number | null; + hits: Array<{ + _source: T; + sort?: [number]; + _index?: string; + _type?: string; + _id?: string; + _score?: number | null; + }>; + }; + }; +} diff --git a/x-pack/plugins/security_solution/public/network/containers/network_top_n_flow/index.tsx b/x-pack/plugins/security_solution/public/network/containers/network_top_n_flow/index.tsx index 770574b0813c1..86e366b253dbe 100644 --- a/x-pack/plugins/security_solution/public/network/containers/network_top_n_flow/index.tsx +++ b/x-pack/plugins/security_solution/public/network/containers/network_top_n_flow/index.tsx @@ -4,161 +4,203 @@ * you may not use this file except in compliance with the Elastic License. */ -import { getOr } from 'lodash/fp'; -import React from 'react'; -import { Query } from 'react-apollo'; -import { connect } from 'react-redux'; -import { compose } from 'redux'; +import { noop } from 'lodash/fp'; +import { useState, useEffect, useCallback, useRef } from 'react'; +import { shallowEqual, useSelector } from 'react-redux'; +import deepEqual from 'fast-deep-equal'; +import { ESTermQuery } from '../../../../common/typed_json'; import { DEFAULT_INDEX_KEY } from '../../../../common/constants'; +import { inputsModel, State } from '../../../common/store'; +import { useKibana } from '../../../common/lib/kibana'; +import { createFilter } from '../../../common/containers/helpers'; +import { PageInfoPaginated } from '../../../../common/search_strategy/security_solution'; +import { generateTablePaginationOptions } from '../../../common/components/paginated_table/helpers'; +import { networkModel, networkSelectors } from '../../store'; import { FlowTargetSourceDest, - GetNetworkTopNFlowQuery, + NetworkQueries, NetworkTopNFlowEdges, - NetworkTopTablesSortField, - PageInfoPaginated, -} from '../../../graphql/types'; -import { withKibana, WithKibanaProps } from '../../../common/lib/kibana'; -import { inputsModel, inputsSelectors, State } from '../../../common/store'; -import { generateTablePaginationOptions } from '../../../common/components/paginated_table/helpers'; -import { createFilter, getDefaultFetchPolicy } from '../../../common/containers/helpers'; -import { - QueryTemplatePaginated, - QueryTemplatePaginatedProps, -} from '../../../common/containers/query_template_paginated'; -import { networkTopNFlowQuery } from './index.gql_query'; -import { networkModel, networkSelectors } from '../../store'; + NetworkTopNFlowRequestOptions, + NetworkTopNFlowStrategyResponse, +} from '../../../../common/search_strategy/security_solution/network'; +import { AbortError } from '../../../../../../../src/plugins/data/common'; +import * as i18n from './translations'; const ID = 'networkTopNFlowQuery'; export interface NetworkTopNFlowArgs { id: string; - ip?: string; inspect: inputsModel.InspectQuery; isInspected: boolean; - loading: boolean; loadPage: (newActivePage: number) => void; - networkTopNFlow: NetworkTopNFlowEdges[]; pageInfo: PageInfoPaginated; refetch: inputsModel.Refetch; + networkTopNFlow: NetworkTopNFlowEdges[]; totalCount: number; } -export interface OwnProps extends QueryTemplatePaginatedProps { - children: (args: NetworkTopNFlowArgs) => React.ReactNode; +interface UseNetworkTopNFlow { flowTarget: FlowTargetSourceDest; ip?: string; type: networkModel.NetworkType; + filterQuery?: ESTermQuery | string; + endDate: string; + startDate: string; + skip: boolean; + id?: string; } -export interface NetworkTopNFlowComponentReduxProps { - activePage: number; - isInspected: boolean; - limit: number; - sort: NetworkTopTablesSortField; -} +export const useNetworkTopNFlow = ({ + endDate, + filterQuery, + flowTarget, + id = ID, + skip, + startDate, + type, +}: UseNetworkTopNFlow): [boolean, NetworkTopNFlowArgs] => { + // const getQuery = inputsSelectors.globalQueryByIdSelector(); + // const { isInspected } = useSelector((state: State) => getQuery(state, id), shallowEqual); + const getTopNFlowSelector = networkSelectors.topNFlowSelector(); + const { activePage, limit, sort } = useSelector( + (state: State) => getTopNFlowSelector(state, type, flowTarget), + shallowEqual + ); + const { data, notifications, uiSettings } = useKibana().services; + const refetch = useRef(noop); + const abortCtrl = useRef(new AbortController()); + const defaultIndex = uiSettings.get(DEFAULT_INDEX_KEY); + const [loading, setLoading] = useState(false); -type NetworkTopNFlowProps = OwnProps & NetworkTopNFlowComponentReduxProps & WithKibanaProps; + const [networkTopNFlowRequest, setTopNFlowRequest] = useState({ + defaultIndex, + factoryQueryType: NetworkQueries.topNFlow, + filterQuery: createFilter(filterQuery), + flowTarget, + // inspect: isInspected, + pagination: generateTablePaginationOptions(activePage, limit), + networkTopNFlowSort: sort, + timerange: { + interval: '12h', + from: startDate ? startDate : '', + to: endDate ? endDate : new Date(Date.now()).toISOString(), + }, + }); -class NetworkTopNFlowComponentQuery extends QueryTemplatePaginated< - NetworkTopNFlowProps, - GetNetworkTopNFlowQuery.Query, - GetNetworkTopNFlowQuery.Variables -> { - public render() { - const { - activePage, - children, - endDate, - flowTarget, - filterQuery, - kibana, - id = `${ID}-${flowTarget}`, - ip, - isInspected, - limit, - skip, - sourceId, - startDate, - sort, - } = this.props; - const variables: GetNetworkTopNFlowQuery.Variables = { - defaultIndex: kibana.services.uiSettings.get(DEFAULT_INDEX_KEY), - filterQuery: createFilter(filterQuery), - flowTarget, - inspect: isInspected, - ip, - pagination: generateTablePaginationOptions(activePage, limit), - sort, - sourceId, - timerange: { - interval: '12h', - from: startDate!, - to: endDate!, - }, - }; - return ( - - fetchPolicy={getDefaultFetchPolicy()} - notifyOnNetworkStatusChange - query={networkTopNFlowQuery} - skip={skip} - variables={variables} - > - {({ data, loading, fetchMore, networkStatus, refetch }) => { - const networkTopNFlow = getOr([], `source.NetworkTopNFlow.edges`, data); - this.setFetchMore(fetchMore); - this.setFetchMoreOptions((newActivePage: number) => ({ - variables: { - pagination: generateTablePaginationOptions(newActivePage, limit), + const wrappedLoadMore = useCallback( + (newActivePage: number) => { + setTopNFlowRequest((prevRequest) => ({ + ...prevRequest, + pagination: generateTablePaginationOptions(newActivePage, limit), + })); + }, + [limit] + ); + + const [networkTopNFlowResponse, setNetworkTopNFlowResponse] = useState({ + networkTopNFlow: [], + id: ID, + inspect: { + dsl: [], + response: [], + }, + isInspected: false, + loadPage: wrappedLoadMore, + pageInfo: { + activePage: 0, + fakeTotalCount: 0, + showMorePagesIndicator: false, + }, + refetch: refetch.current, + totalCount: -1, + }); + + const networkTopNFlowSearch = useCallback( + (request: NetworkTopNFlowRequestOptions) => { + let didCancel = false; + const asyncSearch = async () => { + abortCtrl.current = new AbortController(); + setLoading(true); + + const searchSubscription$ = data.search + .search(request, { + strategy: 'securitySolutionSearchStrategy', + signal: abortCtrl.current.signal, + }) + .subscribe({ + next: (response) => { + if (!response.isPartial && !response.isRunning) { + if (!didCancel) { + setLoading(false); + setNetworkTopNFlowResponse((prevResponse) => ({ + ...prevResponse, + networkTopNFlow: response.edges, + inspect: response.inspect ?? prevResponse.inspect, + pageInfo: response.pageInfo, + refetch: refetch.current, + totalCount: response.totalCount, + })); + } + searchSubscription$.unsubscribe(); + } else if (response.isPartial && !response.isRunning) { + if (!didCancel) { + setLoading(false); + } + // TODO: Make response error status clearer + notifications.toasts.addWarning(i18n.ERROR_NETWORK_TOP_N_FLOW); + searchSubscription$.unsubscribe(); + } }, - updateQuery: (prev, { fetchMoreResult }) => { - if (!fetchMoreResult) { - return prev; + error: (msg) => { + if (!(msg instanceof AbortError)) { + notifications.toasts.addDanger({ + title: i18n.FAIL_NETWORK_TOP_N_FLOW, + text: msg.message, + }); } - return { - ...fetchMoreResult, - source: { - ...fetchMoreResult.source, - NetworkTopNFlow: { - ...fetchMoreResult.source.NetworkTopNFlow, - edges: [...fetchMoreResult.source.NetworkTopNFlow.edges], - }, - }, - }; }, - })); - const isLoading = this.isItAValidLoading(loading, variables, networkStatus); - return children({ - id, - inspect: getOr(null, 'source.NetworkTopNFlow.inspect', data), - isInspected, - loading: isLoading, - loadPage: this.wrappedLoadMore, - networkTopNFlow, - pageInfo: getOr({}, 'source.NetworkTopNFlow.pageInfo', data), - refetch: this.memoizedRefetchQuery(variables, limit, refetch), - totalCount: getOr(-1, 'source.NetworkTopNFlow.totalCount', data), }); - }} - - ); - } -} + }; + abortCtrl.current.abort(); + asyncSearch(); + refetch.current = asyncSearch; + return () => { + didCancel = true; + abortCtrl.current.abort(); + }; + }, + [data.search, notifications.toasts] + ); -const makeMapStateToProps = () => { - const getTopNFlowSelector = networkSelectors.topNFlowSelector(); - const getQuery = inputsSelectors.globalQueryByIdSelector(); - return (state: State, { flowTarget, id = `${ID}-${flowTarget}`, type }: OwnProps) => { - const { isInspected } = getQuery(state, id); - return { - ...getTopNFlowSelector(state, type, flowTarget), - isInspected, - }; - }; -}; + useEffect(() => { + if (skip) { + return; + } + + setTopNFlowRequest((prevRequest) => { + const myRequest = { + ...prevRequest, + defaultIndex, + filterQuery: createFilter(filterQuery), + pagination: generateTablePaginationOptions(activePage, limit), + timerange: { + interval: '12h', + from: startDate, + to: endDate, + }, + networkTopNFlowSort: sort, + }; + if (!deepEqual(prevRequest, myRequest)) { + return myRequest; + } + return prevRequest; + }); + }, [activePage, defaultIndex, endDate, filterQuery, limit, startDate, sort, skip]); -export const NetworkTopNFlowQuery = compose>( - connect(makeMapStateToProps), - withKibana -)(NetworkTopNFlowComponentQuery); + useEffect(() => { + networkTopNFlowSearch(networkTopNFlowRequest); + }, [networkTopNFlowRequest, networkTopNFlowSearch]); + + return [loading, networkTopNFlowResponse]; +}; diff --git a/x-pack/plugins/security_solution/public/network/containers/network_top_n_flow/translations.ts b/x-pack/plugins/security_solution/public/network/containers/network_top_n_flow/translations.ts new file mode 100644 index 0000000000000..4ea704571cf2e --- /dev/null +++ b/x-pack/plugins/security_solution/public/network/containers/network_top_n_flow/translations.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { i18n } from '@kbn/i18n'; + +export const ERROR_NETWORK_TOP_N_FLOW = i18n.translate( + 'xpack.securitySolution.networkTopNFlow.errorSearchDescription', + { + defaultMessage: `An error has occurred on network top n flow search`, + } +); + +export const FAIL_NETWORK_TOP_N_FLOW = i18n.translate( + 'xpack.securitySolution.networkTopNFlow.failSearchDescription', + { + defaultMessage: `Failed to run search on network top n flow`, + } +); diff --git a/x-pack/plugins/security_solution/public/network/pages/ip_details/network_top_n_flow_query_table.tsx b/x-pack/plugins/security_solution/public/network/pages/ip_details/network_top_n_flow_query_table.tsx index 158b4057a7d5e..821452201b78b 100644 --- a/x-pack/plugins/security_solution/public/network/pages/ip_details/network_top_n_flow_query_table.tsx +++ b/x-pack/plugins/security_solution/public/network/pages/ip_details/network_top_n_flow_query_table.tsx @@ -8,7 +8,7 @@ import { getOr } from 'lodash/fp'; import React from 'react'; import { manageQuery } from '../../../common/components/page/manage_query'; import { NetworkTopNFlowTable } from '../../components/network_top_n_flow_table'; -import { NetworkTopNFlowQuery } from '../../containers/network_top_n_flow'; +import { useNetworkTopNFlow } from '../../containers/network_top_n_flow'; import { NetworkWithIndexComponentsQueryTableProps } from './types'; const NetworkTopNFlowTableManage = manageQuery(NetworkTopNFlowTable); @@ -22,45 +22,37 @@ export const NetworkTopNFlowQueryTable = ({ skip, startDate, type, -}: NetworkWithIndexComponentsQueryTableProps) => ( - - {({ - id, - inspect, - isInspected, - loading, - loadPage, - networkTopNFlow, - pageInfo, - refetch, - totalCount, - }) => ( - - )} - -); +}: NetworkWithIndexComponentsQueryTableProps) => { + const [ + loading, + { id, inspect, isInspected, loadPage, networkTopNFlow, pageInfo, refetch, totalCount }, + ] = useNetworkTopNFlow({ + endDate, + filterQuery, + flowTarget, + ip, + skip, + startDate, + type, + }); + + return ( + + ); +}; NetworkTopNFlowQueryTable.displayName = 'NetworkTopNFlowQueryTable'; diff --git a/x-pack/plugins/security_solution/public/network/pages/navigation/ips_query_tab_body.tsx b/x-pack/plugins/security_solution/public/network/pages/navigation/ips_query_tab_body.tsx index a9f4d504847a0..c83bf6ff80901 100644 --- a/x-pack/plugins/security_solution/public/network/pages/navigation/ips_query_tab_body.tsx +++ b/x-pack/plugins/security_solution/public/network/pages/navigation/ips_query_tab_body.tsx @@ -8,7 +8,7 @@ import React from 'react'; import { getOr } from 'lodash/fp'; import { NetworkTopNFlowTable } from '../../components/network_top_n_flow_table'; -import { NetworkTopNFlowQuery } from '../../containers/network_top_n_flow'; +import { useNetworkTopNFlow } from '../../containers/network_top_n_flow'; import { networkModel } from '../../store'; import { manageQuery } from '../../../common/components/page/manage_query'; @@ -23,44 +23,36 @@ export const IPsQueryTabBody = ({ startDate, setQuery, flowTarget, -}: IPsQueryTabBodyProps) => ( - - {({ - id, - inspect, - isInspected, - loading, - loadPage, - networkTopNFlow, - pageInfo, - refetch, - totalCount, - }) => ( - - )} - -); +}: IPsQueryTabBodyProps) => { + const [ + loading, + { id, inspect, isInspected, loadPage, networkTopNFlow, pageInfo, refetch, totalCount }, + ] = useNetworkTopNFlow({ + endDate, + flowTarget, + filterQuery, + skip, + startDate, + type: networkModel.NetworkType.page, + }); + + return ( + + ); +}; IPsQueryTabBody.displayName = 'IPsQueryTabBody'; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/helpers.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/helpers.ts index a7fba087b87ed..c6393e6a289d7 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/helpers.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/helpers.ts @@ -5,7 +5,12 @@ */ import { assertUnreachable } from '../../../../../common/utility_types'; -import { FlowTargetSourceDest } from '../../../../../common/search_strategy/security_solution/network'; +import { + Direction, + NetworkTopTablesSortField, + FlowTargetSourceDest, + NetworkTopTablesFields, +} from '../../../../../common/search_strategy/security_solution/network'; export const getOppositeField = (flowTarget: FlowTargetSourceDest): FlowTargetSourceDest => { switch (flowTarget) { @@ -16,3 +21,28 @@ export const getOppositeField = (flowTarget: FlowTargetSourceDest): FlowTargetSo } assertUnreachable(flowTarget); }; + +type QueryOrder = + | { bytes_in: Direction } + | { bytes_out: Direction } + | { flows: Direction } + | { destination_ips: Direction } + | { source_ips: Direction }; + +export const getQueryOrder = ( + networkTopCountriesSortField: NetworkTopTablesSortField +): QueryOrder => { + switch (networkTopCountriesSortField.field) { + case NetworkTopTablesFields.bytes_in: + return { bytes_in: networkTopCountriesSortField.direction }; + case NetworkTopTablesFields.bytes_out: + return { bytes_out: networkTopCountriesSortField.direction }; + case NetworkTopTablesFields.flows: + return { flows: networkTopCountriesSortField.direction }; + case NetworkTopTablesFields.destination_ips: + return { destination_ips: networkTopCountriesSortField.direction }; + case NetworkTopTablesFields.source_ips: + return { source_ips: networkTopCountriesSortField.direction }; + } + assertUnreachable(networkTopCountriesSortField.field); +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/index.ts index 93e5f113197da..2561ec1de48e2 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/index.ts @@ -11,9 +11,11 @@ import { SecuritySolutionFactory } from '../types'; import { networkHttp } from './http'; import { networkTls } from './tls'; import { networkTopCountries } from './top_countries'; +import { networkTopNFlow } from './top_n_flow'; export const networkFactory: Record> = { [NetworkQueries.http]: networkHttp, [NetworkQueries.tls]: networkTls, [NetworkQueries.topCountries]: networkTopCountries, + [NetworkQueries.topNFlow]: networkTopNFlow, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/helpers.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/helpers.ts new file mode 100644 index 0000000000000..098fd43cd03fa --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/helpers.ts @@ -0,0 +1,97 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { getOr } from 'lodash/fp'; + +import { IEsSearchResponse } from '../../../../../../../../../src/plugins/data/common'; +import { + GeoItem, + NetworkTopNFlowBuckets, + NetworkTopNFlowEdges, + NetworkTopNFlowRequestOptions, + FlowTargetSourceDest, + AutonomousSystemItem, +} from '../../../../../../common/search_strategy/security_solution/network'; +import { getOppositeField } from '../helpers'; + +export const getTopNFlowEdges = ( + response: IEsSearchResponse, + options: NetworkTopNFlowRequestOptions +): NetworkTopNFlowEdges[] => + formatTopNFlowEdges( + getOr([], `aggregations.${options.flowTarget}.buckets`, response.rawResponse), + options.flowTarget + ); + +const formatTopNFlowEdges = ( + buckets: NetworkTopNFlowBuckets[], + flowTarget: FlowTargetSourceDest +): NetworkTopNFlowEdges[] => + buckets.map((bucket: NetworkTopNFlowBuckets) => ({ + node: { + _id: bucket.key, + [flowTarget]: { + domain: bucket.domain.buckets.map((bucketDomain) => bucketDomain.key), + ip: bucket.key, + location: getGeoItem(bucket), + autonomous_system: getAsItem(bucket), + flows: getOr(0, 'flows.value', bucket), + [`${getOppositeField(flowTarget)}_ips`]: getOr( + 0, + `${getOppositeField(flowTarget)}_ips.value`, + bucket + ), + }, + network: { + bytes_in: getOr(0, 'bytes_in.value', bucket), + bytes_out: getOr(0, 'bytes_out.value', bucket), + }, + }, + cursor: { + value: bucket.key, + tiebreaker: null, + }, + })); + +const getFlowTargetFromString = (flowAsString: string) => + flowAsString === 'source' ? FlowTargetSourceDest.source : FlowTargetSourceDest.destination; + +const getGeoItem = (result: NetworkTopNFlowBuckets): GeoItem | null => + result.location.top_geo.hits.hits.length > 0 && result.location.top_geo.hits.hits[0]._source + ? { + geo: getOr( + '', + `location.top_geo.hits.hits[0]._source.${ + Object.keys(result.location.top_geo.hits.hits[0]._source)[0] + }.geo`, + result + ), + flowTarget: getFlowTargetFromString( + Object.keys(result.location.top_geo.hits.hits[0]._source)[0] + ), + } + : null; + +const getAsItem = (result: NetworkTopNFlowBuckets): AutonomousSystemItem | null => + result.autonomous_system.top_as.hits.hits.length > 0 && + result.autonomous_system.top_as.hits.hits[0]._source + ? { + number: getOr( + null, + `autonomous_system.top_as.hits.hits[0]._source.${ + Object.keys(result.autonomous_system.top_as.hits.hits[0]._source)[0] + }.as.number`, + result + ), + name: getOr( + '', + `autonomous_system.top_as.hits.hits[0]._source.${ + Object.keys(result.autonomous_system.top_as.hits.hits[0]._source)[0] + }.as.organization.name`, + result + ), + } + : null; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/index.ts new file mode 100644 index 0000000000000..b9014f76b8def --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/index.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { getOr } from 'lodash/fp'; + +import { IEsSearchResponse } from '../../../../../../../../../src/plugins/data/common'; + +import { DEFAULT_MAX_TABLE_QUERY_SIZE } from '../../../../../../common/constants'; +import { + NetworkTopNFlowStrategyResponse, + NetworkQueries, + NetworkTopNFlowRequestOptions, + NetworkTopNFlowEdges, +} from '../../../../../../common/search_strategy/security_solution/network'; + +import { inspectStringifyObject } from '../../../../../utils/build_query'; +import { SecuritySolutionFactory } from '../../types'; + +import { getTopNFlowEdges } from './helpers'; +import { buildTopNFlowQuery } from './query.top_n_flow_network.dsl'; + +export const networkTopNFlow: SecuritySolutionFactory = { + buildDsl: (options: NetworkTopNFlowRequestOptions) => { + if (options.pagination && options.pagination.querySize >= DEFAULT_MAX_TABLE_QUERY_SIZE) { + throw new Error(`No query size above ${DEFAULT_MAX_TABLE_QUERY_SIZE}`); + } + return buildTopNFlowQuery(options); + }, + parse: async ( + options: NetworkTopNFlowRequestOptions, + response: IEsSearchResponse + ): Promise => { + const { activePage, cursorStart, fakePossibleCount, querySize } = options.pagination; + const totalCount = getOr(0, 'aggregations.top_n_flow_count.value', response.rawResponse); + const networkTopNFlowEdges: NetworkTopNFlowEdges[] = getTopNFlowEdges(response, options); + const fakeTotalCount = fakePossibleCount <= totalCount ? fakePossibleCount : totalCount; + const edges = networkTopNFlowEdges.splice(cursorStart, querySize - cursorStart); + const inspect = { + dsl: [inspectStringifyObject(buildTopNFlowQuery(options))], + response: [inspectStringifyObject(response)], + }; + const showMorePagesIndicator = totalCount > fakeTotalCount; + + return { + ...response, + edges, + inspect, + pageInfo: { + activePage: activePage ? activePage : 0, + fakeTotalCount, + showMorePagesIndicator, + }, + totalCount, + }; + }, +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/query.top_n_flow_network.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/query.top_n_flow_network.dsl.ts new file mode 100644 index 0000000000000..49c51117d2b05 --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/query.top_n_flow_network.dsl.ts @@ -0,0 +1,155 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + FlowTargetSourceDest, + NetworkTopTablesSortField, + NetworkTopNFlowRequestOptions, +} from '../../../../../../common/search_strategy/security_solution/network'; +import { createQueryFilterClauses } from '../../../../../utils/build_query'; +import { getOppositeField, getQueryOrder } from '../helpers'; + +const getCountAgg = (flowTarget: FlowTargetSourceDest) => ({ + top_n_flow_count: { + cardinality: { + field: `${flowTarget}.ip`, + }, + }, +}); + +export const buildTopNFlowQuery = ({ + defaultIndex, + filterQuery, + flowTarget, + networkTopNFlowSort, + pagination: { querySize }, + timerange: { from, to }, + ip, +}: NetworkTopNFlowRequestOptions) => { + const filter = [ + ...createQueryFilterClauses(filterQuery), + { + range: { + '@timestamp': { gte: from, lte: to, format: 'strict_date_optional_time' }, + }, + }, + ]; + + const dslQuery = { + allowNoIndices: true, + index: defaultIndex, + ignoreUnavailable: true, + body: { + aggregations: { + ...getCountAgg(flowTarget), + ...getFlowTargetAggs(networkTopNFlowSort, flowTarget, querySize), + }, + query: { + bool: ip + ? { + filter, + should: [ + { + term: { + [`${getOppositeField(flowTarget)}.ip`]: ip, + }, + }, + ], + minimum_should_match: 1, + } + : { + filter, + }, + }, + }, + size: 0, + track_total_hits: false, + }; + return dslQuery; +}; + +const getFlowTargetAggs = ( + networkTopNFlowSortField: NetworkTopTablesSortField, + flowTarget: FlowTargetSourceDest, + querySize: number +) => ({ + [flowTarget]: { + terms: { + field: `${flowTarget}.ip`, + size: querySize, + order: { + ...getQueryOrder(networkTopNFlowSortField), + }, + }, + aggs: { + bytes_in: { + sum: { + field: `${getOppositeField(flowTarget)}.bytes`, + }, + }, + bytes_out: { + sum: { + field: `${flowTarget}.bytes`, + }, + }, + domain: { + terms: { + field: `${flowTarget}.domain`, + order: { + timestamp: 'desc', + }, + }, + aggs: { + timestamp: { + max: { + field: '@timestamp', + }, + }, + }, + }, + location: { + filter: { + exists: { + field: `${flowTarget}.geo`, + }, + }, + aggs: { + top_geo: { + top_hits: { + _source: `${flowTarget}.geo.*`, + size: 1, + }, + }, + }, + }, + autonomous_system: { + filter: { + exists: { + field: `${flowTarget}.as`, + }, + }, + aggs: { + top_as: { + top_hits: { + _source: `${flowTarget}.as.*`, + size: 1, + }, + }, + }, + }, + flows: { + cardinality: { + field: 'network.community_id', + }, + }, + [`${getOppositeField(flowTarget)}_ips`]: { + cardinality: { + field: `${getOppositeField(flowTarget)}.ip`, + }, + }, + }, + }, +});