From 7a39e3320c269650231dc21c481d5609d4def2bc Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 13 Nov 2024 11:29:52 +1100 Subject: [PATCH 1/2] [8.x] [Cloud Security] Added filter support to graph API (#199048) (#199702) # Backport This will backport the following commits from `main` to `8.x`: - [[Cloud Security] Added filter support to graph API (#199048)](https://github.com/elastic/kibana/pull/199048) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) Co-authored-by: Kfir Peled <61654899+kfirpeled@users.noreply.github.com> --- .../common/schema/graph/v1.ts | 17 +- .../common/tsconfig.json | 1 + .../common/types/graph/v1.ts | 13 +- .../server/routes/graph/route.ts | 20 +- .../server/routes/graph/types.ts | 23 -- .../server/routes/graph/v1.ts | 273 +++++++++++------- .../components/graph_preview_container.tsx | 1 - .../right/hooks/use_fetch_graph_data.test.tsx | 89 ++++++ .../right/hooks/use_fetch_graph_data.ts | 19 +- .../es_archives/logs_gcp_audit/data.json | 120 ++++++++ .../routes/graph.ts | 272 +++++++++++++++-- .../test/cloud_security_posture_api/utils.ts | 9 +- .../security/cloud_security_posture/graph.ts | 12 +- 13 files changed, 693 insertions(+), 176 deletions(-) delete mode 100644 x-pack/plugins/cloud_security_posture/server/routes/graph/types.ts create mode 100644 x-pack/plugins/security_solution/public/flyout/document_details/right/hooks/use_fetch_graph_data.test.tsx diff --git a/x-pack/packages/kbn-cloud-security-posture/common/schema/graph/v1.ts b/x-pack/packages/kbn-cloud-security-posture/common/schema/graph/v1.ts index 3d37331b4cc5d..076c685aca5b9 100644 --- a/x-pack/packages/kbn-cloud-security-posture/common/schema/graph/v1.ts +++ b/x-pack/packages/kbn-cloud-security-posture/common/schema/graph/v1.ts @@ -6,14 +6,26 @@ */ import { schema } from '@kbn/config-schema'; +import { ApiMessageCode } from '../../types/graph/v1'; export const graphRequestSchema = schema.object({ + nodesLimit: schema.maybe(schema.number()), + showUnknownTarget: schema.maybe(schema.boolean()), query: schema.object({ - actorIds: schema.arrayOf(schema.string()), eventIds: schema.arrayOf(schema.string()), // TODO: use zod for range validation instead of config schema start: schema.oneOf([schema.number(), schema.string()]), end: schema.oneOf([schema.number(), schema.string()]), + esQuery: schema.maybe( + schema.object({ + bool: schema.object({ + filter: schema.maybe(schema.arrayOf(schema.object({}, { unknowns: 'allow' }))), + must: schema.maybe(schema.arrayOf(schema.object({}, { unknowns: 'allow' }))), + should: schema.maybe(schema.arrayOf(schema.object({}, { unknowns: 'allow' }))), + must_not: schema.maybe(schema.arrayOf(schema.object({}, { unknowns: 'allow' }))), + }), + }) + ), }), }); @@ -23,6 +35,9 @@ export const graphResponseSchema = () => schema.oneOf([entityNodeDataSchema, groupNodeDataSchema, labelNodeDataSchema]) ), edges: schema.arrayOf(edgeDataSchema), + messages: schema.maybe( + schema.arrayOf(schema.oneOf([schema.literal(ApiMessageCode.ReachedNodesLimit)])) + ), }); export const colorSchema = schema.oneOf([ diff --git a/x-pack/packages/kbn-cloud-security-posture/common/tsconfig.json b/x-pack/packages/kbn-cloud-security-posture/common/tsconfig.json index c7cf1e9208bfc..ebec9929559f0 100644 --- a/x-pack/packages/kbn-cloud-security-posture/common/tsconfig.json +++ b/x-pack/packages/kbn-cloud-security-posture/common/tsconfig.json @@ -20,5 +20,6 @@ "@kbn/i18n", "@kbn/analytics", "@kbn/usage-collection-plugin", + "@kbn/es-query", ] } diff --git a/x-pack/packages/kbn-cloud-security-posture/common/types/graph/v1.ts b/x-pack/packages/kbn-cloud-security-posture/common/types/graph/v1.ts index 48d1d1c49fd03..f97d11b34732c 100644 --- a/x-pack/packages/kbn-cloud-security-posture/common/types/graph/v1.ts +++ b/x-pack/packages/kbn-cloud-security-posture/common/types/graph/v1.ts @@ -6,6 +6,7 @@ */ import type { TypeOf } from '@kbn/config-schema'; +import type { BoolQuery } from '@kbn/es-query'; import { colorSchema, edgeDataSchema, @@ -17,13 +18,21 @@ import { nodeShapeSchema, } from '../../schema/graph/v1'; -export type GraphRequest = TypeOf; -export type GraphResponse = TypeOf; +export type GraphRequest = Omit, 'query.esQuery'> & { + query: { esQuery?: { bool: Partial } }; +}; +export type GraphResponse = Omit, 'messages'> & { + messages?: ApiMessageCode[]; +}; export type Color = typeof colorSchema.type; export type NodeShape = TypeOf; +export enum ApiMessageCode { + ReachedNodesLimit = 'REACHED_NODES_LIMIT', +} + export type EntityNodeDataModel = TypeOf; export type GroupNodeDataModel = TypeOf; diff --git a/x-pack/plugins/cloud_security_posture/server/routes/graph/route.ts b/x-pack/plugins/cloud_security_posture/server/routes/graph/route.ts index 9e9744b33d940..9fb817b275a0d 100644 --- a/x-pack/plugins/cloud_security_posture/server/routes/graph/route.ts +++ b/x-pack/plugins/cloud_security_posture/server/routes/graph/route.ts @@ -10,6 +10,7 @@ import { graphResponseSchema, } from '@kbn/cloud-security-posture-common/schema/graph/latest'; import { transformError } from '@kbn/securitysolution-es-utils'; +import type { GraphRequest } from '@kbn/cloud-security-posture-common/types/graph/v1'; import { GRAPH_ROUTE_PATH } from '../../../common/constants'; import { CspRouter } from '../../types'; import { getGraph as getGraphV1 } from './v1'; @@ -39,26 +40,29 @@ export const defineGraphRoute = (router: CspRouter) => }, }, async (context, request, response) => { - const { actorIds, eventIds, start, end } = request.body.query; + const { nodesLimit, showUnknownTarget = false } = request.body; + const { eventIds, start, end, esQuery } = request.body.query as GraphRequest['query']; const cspContext = await context.csp; const spaceId = (await cspContext.spaces?.spacesService?.getActiveSpace(request))?.id; try { - const { nodes, edges } = await getGraphV1( - { + const resp = await getGraphV1({ + services: { logger: cspContext.logger, esClient: cspContext.esClient, }, - { - actorIds, + query: { eventIds, spaceId, start, end, - } - ); + esQuery, + }, + showUnknownTarget, + nodesLimit, + }); - return response.ok({ body: { nodes, edges } }); + return response.ok({ body: resp }); } catch (err) { const error = transformError(err); cspContext.logger.error(`Failed to fetch graph ${err}`); diff --git a/x-pack/plugins/cloud_security_posture/server/routes/graph/types.ts b/x-pack/plugins/cloud_security_posture/server/routes/graph/types.ts deleted file mode 100644 index ba32664da6233..0000000000000 --- a/x-pack/plugins/cloud_security_posture/server/routes/graph/types.ts +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { - EdgeDataModel, - NodeDataModel, -} from '@kbn/cloud-security-posture-common/types/graph/latest'; -import type { Logger, IScopedClusterClient } from '@kbn/core/server'; -import type { Writable } from '@kbn/utility-types'; - -export interface GraphContextServices { - logger: Logger; - esClient: IScopedClusterClient; -} - -export interface GraphContext { - nodes: Array>; - edges: Array>; -} diff --git a/x-pack/plugins/cloud_security_posture/server/routes/graph/v1.ts b/x-pack/plugins/cloud_security_posture/server/routes/graph/v1.ts index 5102d153c1905..b14a2ba3e06a9 100644 --- a/x-pack/plugins/cloud_security_posture/server/routes/graph/v1.ts +++ b/x-pack/plugins/cloud_security_posture/server/routes/graph/v1.ts @@ -8,22 +8,27 @@ import { castArray } from 'lodash'; import { v4 as uuidv4 } from 'uuid'; import type { Logger, IScopedClusterClient } from '@kbn/core/server'; +import { ApiMessageCode } from '@kbn/cloud-security-posture-common/types/graph/latest'; import type { + Color, EdgeDataModel, - NodeDataModel, EntityNodeDataModel, - LabelNodeDataModel, + GraphRequest, + GraphResponse, GroupNodeDataModel, -} from '@kbn/cloud-security-posture-common/types/graph/latest'; + LabelNodeDataModel, + NodeDataModel, +} from '@kbn/cloud-security-posture-common/types/graph/v1'; import type { EsqlToRecords } from '@elastic/elasticsearch/lib/helpers'; import type { Writable } from '@kbn/utility-types'; -import type { GraphContextServices, GraphContext } from './types'; + +type EsQuery = GraphRequest['query']['esQuery']; interface GraphEdge { badge: number; - ips: string[]; - hosts: string[]; - users: string[]; + ips?: string[] | string; + hosts?: string[] | string; + users?: string[] | string; actorIds: string[] | string; action: string; targetIds: string[] | string; @@ -36,50 +41,75 @@ interface LabelEdges { target: string; } -export const getGraph = async ( - services: GraphContextServices, +interface GraphContextServices { + logger: Logger; + esClient: IScopedClusterClient; +} + +interface GetGraphParams { + services: GraphContextServices; query: { - actorIds: string[]; eventIds: string[]; spaceId?: string; start: string | number; end: string | number; - } -): Promise<{ - nodes: NodeDataModel[]; - edges: EdgeDataModel[]; -}> => { - const { esClient, logger } = services; - const { actorIds, eventIds, spaceId = 'default', start, end } = query; - - logger.trace( - `Fetching graph for [eventIds: ${eventIds.join(', ')}] [actorIds: ${actorIds.join( - ', ' - )}] in [spaceId: ${spaceId}]` - ); + esQuery?: EsQuery; + }; + showUnknownTarget: boolean; + nodesLimit?: number; +} - const results = await fetchGraph({ esClient, logger, start, end, eventIds, actorIds }); +export const getGraph = async ({ + services: { esClient, logger }, + query: { eventIds, spaceId = 'default', start, end, esQuery }, + showUnknownTarget, + nodesLimit, +}: GetGraphParams): Promise> => { + logger.trace(`Fetching graph for [eventIds: ${eventIds.join(', ')}] in [spaceId: ${spaceId}]`); + + const results = await fetchGraph({ + esClient, + showUnknownTarget, + logger, + start, + end, + eventIds, + esQuery, + }); // Convert results into set of nodes and edges - const graphContext = parseRecords(logger, results.records); - - return { nodes: graphContext.nodes, edges: graphContext.edges }; + return parseRecords(logger, results.records, nodesLimit); }; interface ParseContext { - nodesMap: Record; - edgesMap: Record; - edgeLabelsNodes: Record; - labelEdges: Record; + readonly nodesLimit?: number; + readonly nodesMap: Record; + readonly edgesMap: Record; + readonly edgeLabelsNodes: Record; + readonly labelEdges: Record; + readonly messages: ApiMessageCode[]; + readonly logger: Logger; } -const parseRecords = (logger: Logger, records: GraphEdge[]): GraphContext => { - const ctx: ParseContext = { nodesMap: {}, edgeLabelsNodes: {}, edgesMap: {}, labelEdges: {} }; +const parseRecords = ( + logger: Logger, + records: GraphEdge[], + nodesLimit?: number +): Pick => { + const ctx: ParseContext = { + nodesLimit, + logger, + nodesMap: {}, + edgeLabelsNodes: {}, + edgesMap: {}, + labelEdges: {}, + messages: [], + }; - logger.trace(`Parsing records [length: ${records.length}]`); + logger.trace(`Parsing records [length: ${records.length}] [nodesLimit: ${nodesLimit ?? 'none'}]`); - createNodes(logger, records, ctx); - createEdgesAndGroups(logger, ctx); + createNodes(records, ctx); + createEdgesAndGroups(ctx); logger.trace( `Parsed [nodes: ${Object.keys(ctx.nodesMap).length}, edges: ${ @@ -90,7 +120,11 @@ const parseRecords = (logger: Logger, records: GraphEdge[]): GraphContext => { // Sort groups to be first (fixes minor layout issue) const nodes = sortNodes(ctx.nodesMap); - return { nodes, edges: Object.values(ctx.edgesMap) }; + return { + nodes, + edges: Object.values(ctx.edgesMap), + messages: ctx.messages.length > 0 ? ctx.messages : undefined, + }; }; const fetchGraph = async ({ @@ -98,15 +132,17 @@ const fetchGraph = async ({ logger, start, end, - actorIds, eventIds, + showUnknownTarget, + esQuery, }: { esClient: IScopedClusterClient; logger: Logger; start: string | number; end: string | number; - actorIds: string[]; eventIds: string[]; + showUnknownTarget: boolean; + esQuery?: EsQuery; }): Promise> => { const query = `from logs-* | WHERE event.action IS NOT NULL AND actor.entity.id IS NOT NULL @@ -124,59 +160,84 @@ const fetchGraph = async ({ targetIds = target.entity.id, eventOutcome = event.outcome, isAlert -| LIMIT 1000`; +| LIMIT 1000 +| SORT isAlert DESC`; logger.trace(`Executing query [${query}]`); return await esClient.asCurrentUser.helpers .esql({ columnar: false, - filter: { - bool: { - must: [ + filter: buildDslFilter(eventIds, showUnknownTarget, start, end, esQuery), + query, + // @ts-ignore - types are not up to date + params: [...eventIds.map((id, idx) => ({ [`al_id${idx}`]: id }))], + }) + .toRecords(); +}; + +const buildDslFilter = ( + eventIds: string[], + showUnknownTarget: boolean, + start: string | number, + end: string | number, + esQuery?: EsQuery +) => ({ + bool: { + filter: [ + { + range: { + '@timestamp': { + gte: start, + lte: end, + }, + }, + }, + ...(showUnknownTarget + ? [] + : [ { - range: { - '@timestamp': { - gte: start, - lte: end, - }, + exists: { + field: 'target.entity.id', }, }, + ]), + { + bool: { + should: [ + ...(esQuery?.bool.filter?.length || + esQuery?.bool.must?.length || + esQuery?.bool.should?.length || + esQuery?.bool.must_not?.length + ? [esQuery] + : []), { - bool: { - should: [ - { - terms: { - 'event.id': eventIds, - }, - }, - { - terms: { - 'actor.entity.id': actorIds, - }, - }, - ], - minimum_should_match: 1, + terms: { + 'event.id': eventIds, }, }, ], + minimum_should_match: 1, }, }, - query, - // @ts-ignore - types are not up to date - params: [...eventIds.map((id, idx) => ({ [`al_id${idx}`]: id }))], - }) - .toRecords(); -}; + ], + }, +}); -const createNodes = ( - logger: Logger, - records: GraphEdge[], - context: Omit -) => { +const createNodes = (records: GraphEdge[], context: Omit) => { const { nodesMap, edgeLabelsNodes, labelEdges } = context; for (const record of records) { + if (context.nodesLimit !== undefined && Object.keys(nodesMap).length >= context.nodesLimit) { + context.logger.debug( + `Reached nodes limit [limit: ${context.nodesLimit}] [current: ${ + Object.keys(nodesMap).length + }]` + ); + context.messages.push(ApiMessageCode.ReachedNodesLimit); + break; + } + const { ips, hosts, users, actorIds, action, targetIds, isAlert, eventOutcome } = record; const actorIdsArray = castArray(actorIds); const targetIdsArray = castArray(targetIds); @@ -190,12 +251,6 @@ const createNodes = ( } }); - logger.trace( - `Parsing record [actorIds: ${actorIdsArray.join( - ', ' - )}, action: ${action}, targetIds: ${targetIdsArray.join(', ')}]` - ); - // Create entity nodes [...actorIdsArray, ...targetIdsArray].forEach((id) => { if (nodesMap[id] === undefined) { @@ -203,10 +258,13 @@ const createNodes = ( id, label: unknownTargets.includes(id) ? 'Unknown' : undefined, color: isAlert ? 'danger' : 'primary', - ...determineEntityNodeShape(id, ips ?? [], hosts ?? [], users ?? []), + ...determineEntityNodeShape( + id, + castArray(ips ?? []), + castArray(hosts ?? []), + castArray(users ?? []) + ), }; - - logger.trace(`Creating entity node [${id}]`); } }); @@ -226,8 +284,6 @@ const createNodes = ( shape: 'label', }; - logger.trace(`Creating label node [${labelNode.id}]`); - nodesMap[labelNode.id] = labelNode; edgeLabelsNodes[edgeId].push(labelNode.id); labelEdges[labelNode.id] = { source: actorId, target: targetId }; @@ -278,7 +334,7 @@ const sortNodes = (nodesMap: Record) => { return [...groupNodes, ...otherNodes]; }; -const createEdgesAndGroups = (logger: Logger, context: ParseContext) => { +const createEdgesAndGroups = (context: ParseContext) => { const { edgeLabelsNodes, edgesMap, nodesMap, labelEdges } = context; Object.entries(edgeLabelsNodes).forEach(([edgeId, edgeLabelsIds]) => { @@ -287,7 +343,6 @@ const createEdgesAndGroups = (logger: Logger, context: ParseContext) => { const edgeLabelId = edgeLabelsIds[0]; connectEntitiesAndLabelNode( - logger, edgesMap, nodesMap, labelEdges[edgeLabelId].source, @@ -300,44 +355,47 @@ const createEdgesAndGroups = (logger: Logger, context: ParseContext) => { shape: 'group', }; nodesMap[groupNode.id] = groupNode; + let groupEdgesColor: Color = 'primary'; + + edgeLabelsIds.forEach((edgeLabelId) => { + (nodesMap[edgeLabelId] as Writable).parentId = groupNode.id; + connectEntitiesAndLabelNode(edgesMap, nodesMap, groupNode.id, edgeLabelId, groupNode.id); + + if ((nodesMap[edgeLabelId] as LabelNodeDataModel).color === 'danger') { + groupEdgesColor = 'danger'; + } else if ( + (nodesMap[edgeLabelId] as LabelNodeDataModel).color === 'warning' && + groupEdgesColor !== 'danger' + ) { + // Use warning only if there's no danger color + groupEdgesColor = 'warning'; + } + }); connectEntitiesAndLabelNode( - logger, edgesMap, nodesMap, labelEdges[edgeLabelsIds[0]].source, groupNode.id, - labelEdges[edgeLabelsIds[0]].target + labelEdges[edgeLabelsIds[0]].target, + groupEdgesColor ); - - edgeLabelsIds.forEach((edgeLabelId) => { - (nodesMap[edgeLabelId] as Writable).parentId = groupNode.id; - connectEntitiesAndLabelNode( - logger, - edgesMap, - nodesMap, - groupNode.id, - edgeLabelId, - groupNode.id - ); - }); } }); }; const connectEntitiesAndLabelNode = ( - logger: Logger, edgesMap: Record, nodesMap: Record, sourceNodeId: string, labelNodeId: string, - targetNodeId: string + targetNodeId: string, + colorOverride?: Color ) => { [ - connectNodes(nodesMap, sourceNodeId, labelNodeId), - connectNodes(nodesMap, labelNodeId, targetNodeId), + connectNodes(nodesMap, sourceNodeId, labelNodeId, colorOverride), + connectNodes(nodesMap, labelNodeId, targetNodeId, colorOverride), ].forEach((edge) => { - logger.trace(`Connecting nodes [${edge.source} -> ${edge.target}]`); edgesMap[edge.id] = edge; }); }; @@ -345,7 +403,8 @@ const connectEntitiesAndLabelNode = ( const connectNodes = ( nodesMap: Record, sourceNodeId: string, - targetNodeId: string + targetNodeId: string, + colorOverride?: Color ): EdgeDataModel => { const sourceNode = nodesMap[sourceNodeId]; const targetNode = nodesMap[targetNodeId]; @@ -360,6 +419,6 @@ const connectNodes = ( id: `a(${sourceNodeId})-b(${targetNodeId})`, source: sourceNodeId, target: targetNodeId, - color, + color: colorOverride ?? color, }; }; diff --git a/x-pack/plugins/security_solution/public/flyout/document_details/right/components/graph_preview_container.tsx b/x-pack/plugins/security_solution/public/flyout/document_details/right/components/graph_preview_container.tsx index be65593364593..af9e8dca1f24f 100644 --- a/x-pack/plugins/security_solution/public/flyout/document_details/right/components/graph_preview_container.tsx +++ b/x-pack/plugins/security_solution/public/flyout/document_details/right/components/graph_preview_container.tsx @@ -32,7 +32,6 @@ export const GraphPreviewContainer: React.FC = () => { const graphFetchQuery = useFetchGraphData({ req: { query: { - actorIds: [], eventIds, start: DEFAULT_FROM, end: DEFAULT_TO, diff --git a/x-pack/plugins/security_solution/public/flyout/document_details/right/hooks/use_fetch_graph_data.test.tsx b/x-pack/plugins/security_solution/public/flyout/document_details/right/hooks/use_fetch_graph_data.test.tsx new file mode 100644 index 0000000000000..c22ec0caa82c5 --- /dev/null +++ b/x-pack/plugins/security_solution/public/flyout/document_details/right/hooks/use_fetch_graph_data.test.tsx @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { renderHook } from '@testing-library/react-hooks'; +import { useFetchGraphData } from './use_fetch_graph_data'; + +const mockUseQuery = jest.fn(); + +jest.mock('@tanstack/react-query', () => { + return { + useQuery: (...args: unknown[]) => mockUseQuery(...args), + }; +}); + +describe('useFetchGraphData', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('Should pass default options when options are not provided', () => { + renderHook(() => { + return useFetchGraphData({ + req: { + query: { + eventIds: [], + start: '2021-09-01T00:00:00.000Z', + end: '2021-09-01T23:59:59.999Z', + }, + }, + }); + }); + + expect(mockUseQuery.mock.calls).toHaveLength(1); + expect(mockUseQuery.mock.calls[0][2]).toEqual({ + enabled: true, + refetchOnWindowFocus: true, + }); + }); + + it('Should should not be enabled when enabled set to false', () => { + renderHook(() => { + return useFetchGraphData({ + req: { + query: { + eventIds: [], + start: '2021-09-01T00:00:00.000Z', + end: '2021-09-01T23:59:59.999Z', + }, + }, + options: { + enabled: false, + }, + }); + }); + + expect(mockUseQuery.mock.calls).toHaveLength(1); + expect(mockUseQuery.mock.calls[0][2]).toEqual({ + enabled: false, + refetchOnWindowFocus: true, + }); + }); + + it('Should should not be refetchOnWindowFocus when refetchOnWindowFocus set to false', () => { + renderHook(() => { + return useFetchGraphData({ + req: { + query: { + eventIds: [], + start: '2021-09-01T00:00:00.000Z', + end: '2021-09-01T23:59:59.999Z', + }, + }, + options: { + refetchOnWindowFocus: false, + }, + }); + }); + + expect(mockUseQuery.mock.calls).toHaveLength(1); + expect(mockUseQuery.mock.calls[0][2]).toEqual({ + enabled: true, + refetchOnWindowFocus: false, + }); + }); +}); diff --git a/x-pack/plugins/security_solution/public/flyout/document_details/right/hooks/use_fetch_graph_data.ts b/x-pack/plugins/security_solution/public/flyout/document_details/right/hooks/use_fetch_graph_data.ts index 2304cfb8d4fd2..9a0e270a9b2e0 100644 --- a/x-pack/plugins/security_solution/public/flyout/document_details/right/hooks/use_fetch_graph_data.ts +++ b/x-pack/plugins/security_solution/public/flyout/document_details/right/hooks/use_fetch_graph_data.ts @@ -10,6 +10,7 @@ import type { GraphRequest, GraphResponse, } from '@kbn/cloud-security-posture-common/types/graph/latest'; +import { useMemo } from 'react'; import { EVENT_GRAPH_VISUALIZATION_API } from '../../../../../common/constants'; import { useHttp } from '../../../../common/lib/kibana'; @@ -30,6 +31,11 @@ export interface UseFetchGraphDataParams { * Defaults to true. */ enabled?: boolean; + /** + * If true, the query will refetch on window focus. + * Defaults to true. + */ + refetchOnWindowFocus?: boolean; }; } @@ -61,18 +67,25 @@ export const useFetchGraphData = ({ req, options, }: UseFetchGraphDataParams): UseFetchGraphDataResult => { - const { actorIds, eventIds, start, end } = req.query; + const { eventIds, start, end, esQuery } = req.query; const http = useHttp(); + const QUERY_KEY = useMemo( + () => ['useFetchGraphData', eventIds, start, end, esQuery], + [end, esQuery, eventIds, start] + ); const { isLoading, isError, data } = useQuery( - ['useFetchGraphData', actorIds, eventIds, start, end], + QUERY_KEY, () => { return http.post(EVENT_GRAPH_VISUALIZATION_API, { version: '1', body: JSON.stringify(req), }); }, - options + { + enabled: options?.enabled ?? true, + refetchOnWindowFocus: options?.refetchOnWindowFocus ?? true, + } ); return { diff --git a/x-pack/test/cloud_security_posture_api/es_archives/logs_gcp_audit/data.json b/x-pack/test/cloud_security_posture_api/es_archives/logs_gcp_audit/data.json index 9f536d0bb6dc9..37f7ebdff5fb1 100644 --- a/x-pack/test/cloud_security_posture_api/es_archives/logs_gcp_audit/data.json +++ b/x-pack/test/cloud_security_posture_api/es_archives/logs_gcp_audit/data.json @@ -497,3 +497,123 @@ } } } + +{ + "type": "doc", + "value": { + "data_stream": "logs-gcp.audit-default", + "id": "5", + "index": ".ds-logs-gcp.audit-default-2024.10.07-000001", + "source": { + "@timestamp": "2024-09-01T12:34:56.789Z", + "actor": { + "entity": { + "id": "admin5@example.com" + } + }, + "client": { + "user": { + "email": "admin5@example.com" + } + }, + "cloud": { + "project": { + "id": "your-project-id" + }, + "provider": "gcp" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "google.iam.admin.v1.ListRoles", + "agent_id_status": "missing", + "category": [ + "session", + "network", + "configuration" + ], + "id": "without target", + "ingested": "2024-10-07T17:47:35Z", + "kind": "event", + "outcome": "success", + "provider": "activity", + "type": [ + "end", + "access", + "allowed" + ] + }, + "gcp": { + "audit": { + "authorization_info": [ + { + "granted": true, + "permission": "iam.roles.create", + "resource": "projects/your-project-id" + } + ], + "logentry_operation": { + "id": "operation-0987654321" + }, + "request": { + "@type": "type.googleapis.com/google.iam.admin.v1.CreateRoleRequest", + "parent": "projects/your-project-id", + "role": { + "description": "A custom role with specific permissions", + "includedPermissions": [ + "resourcemanager.projects.get", + "resourcemanager.projects.list" + ], + "name": "projects/your-project-id/roles/customRole", + "title": "Custom Role" + }, + "roleId": "customRole" + }, + "resource_name": "projects/your-project-id/roles/customRole", + "response": { + "@type": "type.googleapis.com/google.iam.admin.v1.Role", + "description": "A custom role with specific permissions", + "includedPermissions": [ + "resourcemanager.projects.get", + "resourcemanager.projects.list" + ], + "name": "projects/your-project-id/roles/customRole", + "stage": "GA", + "title": "Custom Role" + }, + "type": "type.googleapis.com/google.cloud.audit.AuditLog" + } + }, + "log": { + "level": "NOTICE", + "logger": "projects/your-project-id/logs/cloudaudit.googleapis.com%2Factivity" + }, + "related": { + "ip": [ + "10.0.0.1" + ], + "user": [ + "admin3@example.com" + ] + }, + "service": { + "name": "iam.googleapis.com" + }, + "source": { + "ip": "10.0.0.1" + }, + "tags": [ + "_geoip_database_unavailable_GeoLite2-City.mmdb", + "_geoip_database_unavailable_GeoLite2-ASN.mmdb" + ], + "user_agent": { + "device": { + "name": "Other" + }, + "name": "Other", + "original": "google-cloud-sdk/324.0.0" + } + } + } +} diff --git a/x-pack/test/cloud_security_posture_api/routes/graph.ts b/x-pack/test/cloud_security_posture_api/routes/graph.ts index bd2f71ef3b9b2..8043e6e22feb6 100644 --- a/x-pack/test/cloud_security_posture_api/routes/graph.ts +++ b/x-pack/test/cloud_security_posture_api/routes/graph.ts @@ -11,6 +11,8 @@ import { } from '@kbn/core-http-common'; import expect from '@kbn/expect'; import type { Agent } from 'supertest'; +import { ApiMessageCode } from '@kbn/cloud-security-posture-common/types/graph/latest'; +import type { GraphRequest } from '@kbn/cloud-security-posture-common/types/graph/latest'; import { FtrProviderContext } from '../ftr_provider_context'; import { result } from '../utils'; import { CspSecurityCommonProvider } from './helper/user_roles_utilites'; @@ -19,12 +21,13 @@ import { CspSecurityCommonProvider } from './helper/user_roles_utilites'; export default function (providerContext: FtrProviderContext) { const { getService } = providerContext; + const logger = getService('log'); const supertest = getService('supertest'); const esArchiver = getService('esArchiver'); const supertestWithoutAuth = getService('supertestWithoutAuth'); const cspSecurity = CspSecurityCommonProvider(providerContext); - const postGraph = (agent: Agent, body: any, auth?: { user: string; pass: string }) => { + const postGraph = (agent: Agent, body: GraphRequest, auth?: { user: string; pass: string }) => { const req = agent .post('/internal/cloud_security_posture/graph') .set(ELASTIC_HTTP_VERSION_HEADER, '1') @@ -45,7 +48,6 @@ export default function (providerContext: FtrProviderContext) { supertestWithoutAuth, { query: { - actorIds: [], eventIds: [], start: 'now-1d/d', end: 'now/d', @@ -55,19 +57,7 @@ export default function (providerContext: FtrProviderContext) { user: 'role_security_no_read_user', pass: cspSecurity.getPasswordForUser('role_security_no_read_user'), } - ).expect(result(403)); - }); - }); - - describe('Validation', () => { - it('should return 400 when missing `actorIds` field', async () => { - await postGraph(supertest, { - query: { - eventIds: [], - start: 'now-1d/d', - end: 'now/d', - }, - }).expect(result(400)); + ).expect(result(403, logger)); }); }); @@ -84,10 +74,54 @@ export default function (providerContext: FtrProviderContext) { ); }); - it('should return an empty graph', async () => { + describe('Validation', () => { + it('should return 400 when missing `eventIds` field', async () => { + await postGraph(supertest, { + // @ts-expect-error ignore error for testing + query: { + start: 'now-1d/d', + end: 'now/d', + }, + }).expect(result(400, logger)); + }); + + it('should return 400 when missing `esQuery` field is not of type bool', async () => { + await postGraph(supertest, { + query: { + eventIds: [], + start: 'now-1d/d', + end: 'now/d', + esQuery: { + // @ts-expect-error ignore error for testing + match_all: {}, + }, + }, + }).expect(result(400, logger)); + }); + + it('should return 400 with unsupported `esQuery`', async () => { + await postGraph(supertest, { + query: { + eventIds: [], + start: 'now-1d/d', + end: 'now/d', + esQuery: { + bool: { + filter: [ + { + unsupported: 'unsupported', + }, + ], + }, + }, + }, + }).expect(result(400, logger)); + }); + }); + + it('should return an empty graph / should return 200 when missing `esQuery` field', async () => { const response = await postGraph(supertest, { query: { - actorIds: [], eventIds: [], start: 'now-1d/d', end: 'now/d', @@ -96,20 +130,32 @@ export default function (providerContext: FtrProviderContext) { expect(response.body).to.have.property('nodes').length(0); expect(response.body).to.have.property('edges').length(0); + expect(response.body).not.to.have.property('messages'); }); it('should return a graph with nodes and edges by actor', async () => { const response = await postGraph(supertest, { query: { - actorIds: ['admin@example.com'], eventIds: [], start: '2024-09-01T00:00:00Z', end: '2024-09-02T00:00:00Z', + esQuery: { + bool: { + filter: [ + { + match_phrase: { + 'actor.entity.id': 'admin@example.com', + }, + }, + ], + }, + }, }, }).expect(result(200)); expect(response.body).to.have.property('nodes').length(3); expect(response.body).to.have.property('edges').length(2); + expect(response.body).not.to.have.property('messages'); response.body.nodes.forEach((node: any) => { expect(node).to.have.property('color'); @@ -131,7 +177,6 @@ export default function (providerContext: FtrProviderContext) { it('should return a graph with nodes and edges by alert', async () => { const response = await postGraph(supertest, { query: { - actorIds: [], eventIds: ['kabcd1234efgh5678'], start: '2024-09-01T00:00:00Z', end: '2024-09-02T00:00:00Z', @@ -140,6 +185,7 @@ export default function (providerContext: FtrProviderContext) { expect(response.body).to.have.property('nodes').length(3); expect(response.body).to.have.property('edges').length(2); + expect(response.body).not.to.have.property('messages'); response.body.nodes.forEach((node: any) => { expect(node).to.have.property('color'); @@ -161,7 +207,6 @@ export default function (providerContext: FtrProviderContext) { it('color of alert of failed event should be danger', async () => { const response = await postGraph(supertest, { query: { - actorIds: [], eventIds: ['failed-event'], start: '2024-09-01T00:00:00Z', end: '2024-09-02T00:00:00Z', @@ -170,6 +215,7 @@ export default function (providerContext: FtrProviderContext) { expect(response.body).to.have.property('nodes').length(3); expect(response.body).to.have.property('edges').length(2); + expect(response.body).not.to.have.property('messages'); response.body.nodes.forEach((node: any) => { expect(node).to.have.property('color'); @@ -191,15 +237,26 @@ export default function (providerContext: FtrProviderContext) { it('color of event of failed event should be warning', async () => { const response = await postGraph(supertest, { query: { - actorIds: ['admin2@example.com'], eventIds: [], start: '2024-09-01T00:00:00Z', end: '2024-09-02T00:00:00Z', + esQuery: { + bool: { + filter: [ + { + match_phrase: { + 'actor.entity.id': 'admin2@example.com', + }, + }, + ], + }, + }, }, }).expect(result(200)); expect(response.body).to.have.property('nodes').length(3); expect(response.body).to.have.property('edges').length(2); + expect(response.body).not.to.have.property('messages'); response.body.nodes.forEach((node: any) => { expect(node).to.have.property('color'); @@ -219,18 +276,29 @@ export default function (providerContext: FtrProviderContext) { }); }); - it('2 grouped of events, 1 failed, 1 success', async () => { + it('2 grouped events, 1 failed, 1 success', async () => { const response = await postGraph(supertest, { query: { - actorIds: ['admin3@example.com'], eventIds: [], start: '2024-09-01T00:00:00Z', end: '2024-09-02T00:00:00Z', + esQuery: { + bool: { + filter: [ + { + match_phrase: { + 'actor.entity.id': 'admin3@example.com', + }, + }, + ], + }, + }, }, }).expect(result(200)); expect(response.body).to.have.property('nodes').length(5); expect(response.body).to.have.property('edges').length(6); + expect(response.body).not.to.have.property('messages'); expect(response.body.nodes[0].shape).equal('group', 'Groups should be the first nodes'); @@ -247,11 +315,167 @@ export default function (providerContext: FtrProviderContext) { response.body.edges.forEach((edge: any) => { expect(edge).to.have.property('color'); expect(edge.color).equal( - edge.id.includes('outcome(failed)') ? 'warning' : 'primary', + edge.id.includes('outcome(failed)') || + (edge.id.includes('grp(') && !edge.id.includes('outcome(success)')) + ? 'warning' + : 'primary', + `edge color mismatched [edge: ${edge.id}] [actual: ${edge.color}]` + ); + }); + }); + + it('should support more than 1 eventIds', async () => { + const response = await postGraph(supertest, { + query: { + eventIds: ['kabcd1234efgh5678', 'failed-event'], + start: '2024-09-01T00:00:00Z', + end: '2024-09-02T00:00:00Z', + }, + }).expect(result(200)); + + expect(response.body).to.have.property('nodes').length(5); + expect(response.body).to.have.property('edges').length(4); + expect(response.body).not.to.have.property('messages'); + + response.body.nodes.forEach((node: any) => { + expect(node).to.have.property('color'); + expect(node.color).equal( + 'danger', + `node color mismatched [node: ${node.id}] [actual: ${node.color}]` + ); + }); + + response.body.edges.forEach((edge: any) => { + expect(edge).to.have.property('color'); + expect(edge.color).equal( + 'danger', + `edge color mismatched [edge: ${edge.id}] [actual: ${edge.color}]` + ); + }); + }); + + it('should return a graph with nodes and edges by alert and actor', async () => { + const response = await postGraph(supertest, { + query: { + eventIds: ['kabcd1234efgh5678'], + start: '2024-09-01T00:00:00Z', + end: '2024-09-02T00:00:00Z', + esQuery: { + bool: { + filter: [ + { + match_phrase: { + 'actor.entity.id': 'admin2@example.com', + }, + }, + ], + }, + }, + }, + }).expect(result(200)); + + expect(response.body).to.have.property('nodes').length(5); + expect(response.body).to.have.property('edges').length(4); + expect(response.body).not.to.have.property('messages'); + + response.body.nodes.forEach((node: any, idx: number) => { + expect(node).to.have.property('color'); + expect(node.color).equal( + idx <= 2 // First 3 nodes are expected to be colored as danger (ORDER MATTERS, alerts are expected to be first) + ? 'danger' + : node.shape === 'label' && node.id.includes('outcome(failed)') + ? 'warning' + : 'primary', + `node color mismatched [node: ${node.id}] [actual: ${node.color}]` + ); + }); + + response.body.edges.forEach((edge: any, idx: number) => { + expect(edge).to.have.property('color'); + expect(edge.color).equal( + idx <= 1 ? 'danger' : 'warning', `edge color mismatched [edge: ${edge.id}] [actual: ${edge.color}]` ); }); }); + + it('Should filter unknown targets', async () => { + const response = await postGraph(supertest, { + query: { + eventIds: [], + start: '2024-09-01T00:00:00Z', + end: '2024-09-02T00:00:00Z', + esQuery: { + bool: { + filter: [ + { + match_phrase: { + 'actor.entity.id': 'admin5@example.com', + }, + }, + ], + }, + }, + }, + }).expect(result(200)); + + expect(response.body).to.have.property('nodes').length(0); + expect(response.body).to.have.property('edges').length(0); + expect(response.body).not.to.have.property('messages'); + }); + + it('Should return unknown targets', async () => { + const response = await postGraph(supertest, { + showUnknownTarget: true, + query: { + eventIds: [], + start: '2024-09-01T00:00:00Z', + end: '2024-09-02T00:00:00Z', + esQuery: { + bool: { + filter: [ + { + match_phrase: { + 'actor.entity.id': 'admin5@example.com', + }, + }, + ], + }, + }, + }, + }).expect(result(200)); + + expect(response.body).to.have.property('nodes').length(3); + expect(response.body).to.have.property('edges').length(2); + expect(response.body).not.to.have.property('messages'); + }); + + it('Should limit number of nodes', async () => { + const response = await postGraph(supertest, { + nodesLimit: 1, + query: { + eventIds: [], + start: '2024-09-01T00:00:00Z', + end: '2024-09-02T00:00:00Z', + esQuery: { + bool: { + filter: [ + { + exists: { + field: 'actor.entity.id', + }, + }, + ], + }, + }, + }, + }).expect(result(200)); + + expect(response.body).to.have.property('nodes').length(3); // Minimal number of nodes in a single relationship + expect(response.body).to.have.property('edges').length(2); + expect(response.body).to.have.property('messages').length(1); + expect(response.body.messages[0]).equal(ApiMessageCode.ReachedNodesLimit); + }); }); }); } diff --git a/x-pack/test/cloud_security_posture_api/utils.ts b/x-pack/test/cloud_security_posture_api/utils.ts index e64c583af3868..210a081b91473 100644 --- a/x-pack/test/cloud_security_posture_api/utils.ts +++ b/x-pack/test/cloud_security_posture_api/utils.ts @@ -36,22 +36,23 @@ export const waitForPluginInitialized = ({ logger.debug('CSP plugin is initialized'); }); -export function result(status: number): CallbackHandler { +export function result(status: number, logger?: ToolingLog): CallbackHandler { return (err: any, res: Response) => { if ((res?.status || err.status) !== status) { - const e = new Error( + throw new Error( `Expected ${status} ,got ${res?.status || err.status} resp: ${ res?.body ? JSON.stringify(res.body) : err.text }` ); - throw e; + } else if (err) { + logger?.warning(`Error result ${err.text}`); } }; } export class EsIndexDataProvider { private es: EsClient; - private index: string; + private readonly index: string; constructor(es: EsClient, index: string) { this.es = es; diff --git a/x-pack/test_serverless/api_integration/test_suites/security/cloud_security_posture/graph.ts b/x-pack/test_serverless/api_integration/test_suites/security/cloud_security_posture/graph.ts index 741d25291e8fa..aaccdd0e9a41c 100644 --- a/x-pack/test_serverless/api_integration/test_suites/security/cloud_security_posture/graph.ts +++ b/x-pack/test_serverless/api_integration/test_suites/security/cloud_security_posture/graph.ts @@ -12,6 +12,7 @@ import { } from '@kbn/core-http-common'; import { result } from '@kbn/test-suites-xpack/cloud_security_posture_api/utils'; import type { Agent } from 'supertest'; +import type { GraphRequest } from '@kbn/cloud-security-posture-common/types/graph/v1'; import type { FtrProviderContext } from '../../../ftr_provider_context'; export default function ({ getService }: FtrProviderContext) { @@ -19,7 +20,7 @@ export default function ({ getService }: FtrProviderContext) { const roleScopedSupertest = getService('roleScopedSupertest'); let supertestViewer: Pick; - const postGraph = (agent: Pick, body: any) => { + const postGraph = (agent: Pick, body: GraphRequest) => { const req = agent .post('/internal/cloud_security_posture/graph') .set(ELASTIC_HTTP_VERSION_HEADER, '1') @@ -48,7 +49,6 @@ export default function ({ getService }: FtrProviderContext) { it('should return an empty graph', async () => { const response = await postGraph(supertestViewer, { query: { - actorIds: [], eventIds: [], start: 'now-1d/d', end: 'now/d', @@ -57,20 +57,26 @@ export default function ({ getService }: FtrProviderContext) { expect(response.body).to.have.property('nodes').length(0); expect(response.body).to.have.property('edges').length(0); + expect(response.body).not.to.have.property('messages'); }); it('should return a graph with nodes and edges by actor', async () => { const response = await postGraph(supertestViewer, { query: { - actorIds: ['admin@example.com'], eventIds: [], start: '2024-09-01T00:00:00Z', end: '2024-09-02T00:00:00Z', + esQuery: { + bool: { + filter: [{ match_phrase: { 'actor.entity.id': 'admin@example.com' } }], + }, + }, }, }).expect(result(200)); expect(response.body).to.have.property('nodes').length(3); expect(response.body).to.have.property('edges').length(2); + expect(response.body).not.to.have.property('messages'); response.body.nodes.forEach((node: any) => { expect(node).to.have.property('color'); From db3f6f223b79984ba4abed79db20f0ed04ee9d62 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 13 Nov 2024 11:38:45 +1100 Subject: [PATCH 2/2] [8.x] [Dataset Quality] Fix failing test (#199763) (#199884) # Backport This will backport the following commits from `main` to `8.x`: - [[Dataset Quality] Fix failing test (#199763)](https://github.com/elastic/kibana/pull/199763) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) Co-authored-by: mohamedhamed-ahmed --- .../dataset_quality/degraded_field_flyout.ts | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts index 1fe53b2aa7ce7..9fb1f74cbae0e 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts @@ -816,13 +816,12 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { expandedDegradedField: 'cloud.project.id', }); - const applyButton = await testSubjects.find( - 'datasetQualityIncreaseFieldMappingLimitButtonButton' - ); - - await applyButton.click(); - await retry.tryForTime(5000, async () => { + const applyButton = await testSubjects.find( + 'datasetQualityIncreaseFieldMappingLimitButtonButton' + ); + await applyButton.click(); + // Should display the error callout await testSubjects.existOrFail('datasetQualityDetailsNewFieldLimitErrorCallout'); });