From cc8e4d4dca1726fccb9261843985b23a692aef46 Mon Sep 17 00:00:00 2001 From: Jason Rhodes Date: Tue, 6 Jun 2023 15:42:35 -0400 Subject: [PATCH] Add asset.kind handling to assets API (#159065) ## Summary Implicit collection stores segmentation values on `asset.kind` instead of on `asset.type`, like originally planned. This PR makes those changes so that `asset.kind` is a valid filter. It leaves `asset.type` in place for the moment. --- .../plugins/asset_manager/common/types_api.ts | 15 ++++- .../server/lib/get_all_related_assets.test.ts | 48 +++++++++----- .../server/lib/get_all_related_assets.ts | 44 ++++++++----- .../asset_manager/server/lib/get_assets.ts | 22 +++++-- ...ts.ts => get_indirectly_related_assets.ts} | 15 +++-- .../asset_manager/server/lib/sample_assets.ts | 66 +++++++++---------- .../plugins/asset_manager/server/lib/utils.ts | 12 ++++ .../asset_manager/server/routes/assets.ts | 42 ++++++++---- .../server/routes/sample_assets.ts | 28 ++++---- .../apis/asset_manager/tests/assets.ts | 66 +++++++++++++++++++ .../apis/asset_manager/tests/assets_diff.ts | 18 +++-- .../asset_manager/tests/assets_related.ts | 26 ++++---- 12 files changed, 275 insertions(+), 127 deletions(-) rename x-pack/plugins/asset_manager/server/lib/{get_related_assets.ts => get_indirectly_related_assets.ts} (84%) diff --git a/x-pack/plugins/asset_manager/common/types_api.ts b/x-pack/plugins/asset_manager/common/types_api.ts index a72392316565e..a2cdbd38aaa41 100644 --- a/x-pack/plugins/asset_manager/common/types_api.ts +++ b/x-pack/plugins/asset_manager/common/types_api.ts @@ -15,7 +15,17 @@ export const assetTypeRT = rt.union([ export type AssetType = rt.TypeOf; -export type AssetKind = 'cluster' | 'host' | 'pod' | 'container' | 'service'; +export const assetKindRT = rt.union([ + rt.literal('cluster'), + rt.literal('host'), + rt.literal('pod'), + rt.literal('container'), + rt.literal('service'), + rt.literal('alert'), +]); + +export type AssetKind = rt.TypeOf; + export type AssetStatus = | 'CREATING' | 'ACTIVE' @@ -124,10 +134,11 @@ export interface K8sCluster extends WithTimestamp { export interface AssetFilters { type?: AssetType | AssetType[]; - kind?: AssetKind; + kind?: AssetKind | AssetKind[]; ean?: string | string[]; id?: string; typeLike?: string; + kindLike?: string; eanLike?: string; collectionVersion?: number | 'latest' | 'all'; from?: string; diff --git a/x-pack/plugins/asset_manager/server/lib/get_all_related_assets.test.ts b/x-pack/plugins/asset_manager/server/lib/get_all_related_assets.test.ts index 6859bfec31910..47006590a9238 100644 --- a/x-pack/plugins/asset_manager/server/lib/get_all_related_assets.test.ts +++ b/x-pack/plugins/asset_manager/server/lib/get_all_related_assets.test.ts @@ -6,13 +6,13 @@ */ jest.mock('./get_assets', () => ({ getAssets: jest.fn() })); -jest.mock('./get_related_assets', () => ({ getRelatedAssets: jest.fn() })); +jest.mock('./get_indirectly_related_assets', () => ({ getIndirectlyRelatedAssets: jest.fn() })); import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks'; import { v4 as uuid } from 'uuid'; import { AssetWithoutTimestamp } from '../../common/types_api'; import { getAssets } from './get_assets'; // Mocked -import { getRelatedAssets } from './get_related_assets'; // Mocked +import { getIndirectlyRelatedAssets } from './get_indirectly_related_assets'; // Mocked import { getAllRelatedAssets } from './get_all_related_assets'; const esClientMock = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; @@ -20,7 +20,7 @@ const esClientMock = elasticsearchClientMock.createScopedClusterClient().asCurre describe('getAllRelatedAssets', () => { beforeEach(() => { (getAssets as jest.Mock).mockReset(); - (getRelatedAssets as jest.Mock).mockReset(); + (getIndirectlyRelatedAssets as jest.Mock).mockReset(); }); it('throws if it cannot find the primary asset', async () => { @@ -35,7 +35,9 @@ describe('getAllRelatedAssets', () => { (getAssets as jest.Mock).mockResolvedValueOnce([]); // Ensure maxDistance is respected (getAssets as jest.Mock).mockRejectedValueOnce(new Error('Should respect maxDistance')); - (getRelatedAssets as jest.Mock).mockRejectedValueOnce(new Error('Should respect maxDistance')); + (getIndirectlyRelatedAssets as jest.Mock).mockRejectedValueOnce( + new Error('Should respect maxDistance') + ); await expect( getAllRelatedAssets(esClientMock, { @@ -61,10 +63,12 @@ describe('getAllRelatedAssets', () => { (getAssets as jest.Mock).mockResolvedValueOnce([primaryAssetWithoutParents]); // Distance 1 - (getRelatedAssets as jest.Mock).mockResolvedValueOnce([]); + (getIndirectlyRelatedAssets as jest.Mock).mockResolvedValueOnce([]); // Ensure maxDistance is respected (getAssets as jest.Mock).mockRejectedValueOnce(new Error('Should respect maxDistance')); - (getRelatedAssets as jest.Mock).mockRejectedValueOnce(new Error('Should respect maxDistance')); + (getIndirectlyRelatedAssets as jest.Mock).mockRejectedValueOnce( + new Error('Should respect maxDistance') + ); await expect( getAllRelatedAssets(esClientMock, { @@ -82,7 +86,7 @@ describe('getAllRelatedAssets', () => { it('returns the primary and a directly referenced parent', async () => { const parentAsset: AssetWithoutTimestamp = { - 'asset.ean': 'primary-ean', + 'asset.ean': 'parent-ean', 'asset.type': 'k8s.pod', 'asset.kind': 'pod', 'asset.id': uuid(), @@ -100,10 +104,12 @@ describe('getAllRelatedAssets', () => { (getAssets as jest.Mock).mockResolvedValueOnce([primaryAssetWithDirectParent]); // Distance 1 (getAssets as jest.Mock).mockResolvedValueOnce([parentAsset]); - (getRelatedAssets as jest.Mock).mockResolvedValueOnce([]); + (getIndirectlyRelatedAssets as jest.Mock).mockResolvedValueOnce([]); // Ensure maxDistance is respected (getAssets as jest.Mock).mockRejectedValueOnce(new Error('Should respect maxDistance')); - (getRelatedAssets as jest.Mock).mockRejectedValueOnce(new Error('Should respect maxDistance')); + (getIndirectlyRelatedAssets as jest.Mock).mockRejectedValueOnce( + new Error('Should respect maxDistance') + ); await expect( getAllRelatedAssets(esClientMock, { @@ -145,10 +151,12 @@ describe('getAllRelatedAssets', () => { (getAssets as jest.Mock).mockResolvedValueOnce([primaryAssetWithIndirectParent]); // Distance 1 (getAssets as jest.Mock).mockResolvedValueOnce([]); - (getRelatedAssets as jest.Mock).mockResolvedValueOnce([parentAsset]); + (getIndirectlyRelatedAssets as jest.Mock).mockResolvedValueOnce([parentAsset]); // Ensure maxDistance is respected (getAssets as jest.Mock).mockRejectedValueOnce(new Error('Should respect maxDistance')); - (getRelatedAssets as jest.Mock).mockRejectedValueOnce(new Error('Should respect maxDistance')); + (getIndirectlyRelatedAssets as jest.Mock).mockRejectedValueOnce( + new Error('Should respect maxDistance') + ); await expect( getAllRelatedAssets(esClientMock, { @@ -198,10 +206,12 @@ describe('getAllRelatedAssets', () => { (getAssets as jest.Mock).mockResolvedValueOnce([primaryAsset]); // Distance 1 (getAssets as jest.Mock).mockResolvedValueOnce([directlyReferencedParent]); - (getRelatedAssets as jest.Mock).mockResolvedValueOnce([indirectlyReferencedParent]); + (getIndirectlyRelatedAssets as jest.Mock).mockResolvedValueOnce([indirectlyReferencedParent]); // Ensure maxDistance is respected (getAssets as jest.Mock).mockRejectedValueOnce(new Error('Should respect maxDistance')); - (getRelatedAssets as jest.Mock).mockRejectedValueOnce(new Error('Should respect maxDistance')); + (getIndirectlyRelatedAssets as jest.Mock).mockRejectedValueOnce( + new Error('Should respect maxDistance') + ); await expect( getAllRelatedAssets(esClientMock, { @@ -249,10 +259,12 @@ describe('getAllRelatedAssets', () => { // Distance 1 (getAssets as jest.Mock).mockResolvedValueOnce([parentAsset]); // Code should filter out any directly referenced parent from the indirectly referenced parents query - (getRelatedAssets as jest.Mock).mockResolvedValueOnce([]); + (getIndirectlyRelatedAssets as jest.Mock).mockResolvedValueOnce([]); // Ensure maxDistance is respected (getAssets as jest.Mock).mockRejectedValueOnce(new Error('Should respect maxDistance')); - (getRelatedAssets as jest.Mock).mockRejectedValueOnce(new Error('Should respect maxDistance')); + (getIndirectlyRelatedAssets as jest.Mock).mockRejectedValueOnce( + new Error('Should respect maxDistance') + ); await expect( getAllRelatedAssets(esClientMock, { @@ -330,7 +342,7 @@ describe('getAllRelatedAssets', () => { }; // Only using directly referenced parents - (getRelatedAssets as jest.Mock).mockResolvedValue([]); + (getIndirectlyRelatedAssets as jest.Mock).mockResolvedValue([]); // Primary (getAssets as jest.Mock).mockResolvedValueOnce([primaryAsset]); @@ -415,7 +427,7 @@ describe('getAllRelatedAssets', () => { }; // Only using directly referenced parents - (getRelatedAssets as jest.Mock).mockResolvedValue([]); + (getIndirectlyRelatedAssets as jest.Mock).mockResolvedValue([]); // Primary (getAssets as jest.Mock).mockResolvedValueOnce([primaryAsset]); @@ -507,7 +519,7 @@ describe('getAllRelatedAssets', () => { }; // Only using directly referenced parents - (getRelatedAssets as jest.Mock).mockResolvedValue([]); + (getIndirectlyRelatedAssets as jest.Mock).mockResolvedValue([]); // Primary (getAssets as jest.Mock).mockResolvedValueOnce([primaryAsset]); diff --git a/x-pack/plugins/asset_manager/server/lib/get_all_related_assets.ts b/x-pack/plugins/asset_manager/server/lib/get_all_related_assets.ts index 9f2ab5f114f9e..ad8aff78cbb18 100644 --- a/x-pack/plugins/asset_manager/server/lib/get_all_related_assets.ts +++ b/x-pack/plugins/asset_manager/server/lib/get_all_related_assets.ts @@ -7,9 +7,10 @@ import { ElasticsearchClient } from '@kbn/core/server'; import { flatten, without } from 'lodash'; -import { Asset, AssetType, Relation, RelationField } from '../../common/types_api'; +import { debug } from '../../common/debug_log'; +import { Asset, AssetType, AssetKind, Relation, RelationField } from '../../common/types_api'; import { getAssets } from './get_assets'; -import { getRelatedAssets } from './get_related_assets'; +import { getIndirectlyRelatedAssets } from './get_indirectly_related_assets'; import { AssetNotFoundError } from './errors'; import { toArray } from './utils'; @@ -19,6 +20,7 @@ interface GetAllRelatedAssetsOptions { to?: string; relation: Relation; type?: AssetType[]; + kind?: AssetKind[]; maxDistance: number; size: number; } @@ -28,7 +30,7 @@ export async function getAllRelatedAssets( options: GetAllRelatedAssetsOptions ) { // How to put size into this? - const { ean, from, to, relation, maxDistance, type = [] } = options; + const { ean, from, to, relation, maxDistance, kind = [] } = options; const primary = await findPrimary(esClient, { ean, from, to }); @@ -42,10 +44,10 @@ export async function getAllRelatedAssets( to, visitedEans: [primary['asset.ean'], ...relatedAssets.map((asset) => asset['asset.ean'])], }; - // if we enforce the type filter before the last query we'll miss nodes with - // possible edges to the requested types - if (currentDistance === maxDistance && type.length) { - queryOptions.type = type; + // if we enforce the kind filter before the last query we'll miss nodes with + // possible edges to the requested kind values + if (currentDistance === maxDistance && kind.length) { + queryOptions.kind = kind; } const results = flatten( @@ -66,8 +68,8 @@ export async function getAllRelatedAssets( return { primary, - [relation]: type.length - ? relatedAssets.filter((asset) => asset['asset.type'] && type.includes(asset['asset.type'])) + [relation]: kind.length + ? relatedAssets.filter((asset) => asset['asset.kind'] && kind.includes(asset['asset.kind'])) : relatedAssets, }; } @@ -95,36 +97,44 @@ async function findPrimary( type FindRelatedAssetsOptions = Pick< GetAllRelatedAssetsOptions, - 'relation' | 'type' | 'from' | 'to' + 'relation' | 'kind' | 'from' | 'to' > & { visitedEans: string[] }; async function findRelatedAssets( esClient: ElasticsearchClient, primary: Asset, - { relation, from, to, type, visitedEans }: FindRelatedAssetsOptions + { relation, from, to, kind, visitedEans }: FindRelatedAssetsOptions ): Promise { const relationField = relationToDirectField(relation); - const directlyRelatedEans = toArray(primary[relationField]); + const directlyRelatedEans = toArray(primary[relationField]); + + debug('Directly Related EAN values found on primary asset', directlyRelatedEans); let directlyRelatedAssets: Asset[] = []; - if (directlyRelatedEans.length) { - // get the directly related assets we haven't visited already + + // get the directly related assets we haven't visited already + const remainingEansToFind = without(directlyRelatedEans, ...visitedEans); + if (remainingEansToFind.length > 0) { directlyRelatedAssets = await getAssets({ esClient, - filters: { ean: without(directlyRelatedEans, ...visitedEans), from, to, type }, + filters: { ean: remainingEansToFind, from, to, kind }, }); } - const indirectlyRelatedAssets = await getRelatedAssets({ + debug('Directly related assets found:', JSON.stringify(directlyRelatedAssets)); + + const indirectlyRelatedAssets = await getIndirectlyRelatedAssets({ esClient, ean: primary['asset.ean'], excludeEans: visitedEans.concat(directlyRelatedEans), relation, from, to, - type, + kind, }); + debug('Indirectly related assets found:', JSON.stringify(indirectlyRelatedAssets)); + return [...directlyRelatedAssets, ...indirectlyRelatedAssets]; } diff --git a/x-pack/plugins/asset_manager/server/lib/get_assets.ts b/x-pack/plugins/asset_manager/server/lib/get_assets.ts index a5c797cdfdfd3..09bcd74ef6c22 100644 --- a/x-pack/plugins/asset_manager/server/lib/get_assets.ts +++ b/x-pack/plugins/asset_manager/server/lib/get_assets.ts @@ -9,6 +9,7 @@ import { debug } from '../../common/debug_log'; import { Asset, AssetFilters } from '../../common/types_api'; import { ASSETS_INDEX_PREFIX } from '../constants'; import { ElasticsearchAccessorOptions } from '../types'; +import { isStringOrNonEmptyArray } from './utils'; interface GetAssetsOptions extends ElasticsearchAccessorOptions { size?: number; @@ -23,6 +24,7 @@ export async function getAssets({ filters = {}, }: GetAssetsOptions): Promise { // Maybe it makes the most sense to validate the filters here? + debug('Get Assets Filters:', JSON.stringify(filters)); const { from = 'now-24h', to = 'now' } = filters; const must: QueryDslQueryContainer[] = []; @@ -36,7 +38,7 @@ export async function getAssets({ }); } - if (filters.type?.length) { + if (isStringOrNonEmptyArray(filters.type)) { must.push({ terms: { ['asset.type']: Array.isArray(filters.type) ? filters.type : [filters.type], @@ -44,15 +46,15 @@ export async function getAssets({ }); } - if (filters.kind) { + if (isStringOrNonEmptyArray(filters.kind)) { must.push({ - term: { - ['asset.kind']: filters.kind, + terms: { + ['asset.kind']: Array.isArray(filters.kind) ? filters.kind : [filters.kind], }, }); } - if (filters.ean) { + if (isStringOrNonEmptyArray(filters.ean)) { must.push({ terms: { ['asset.ean']: Array.isArray(filters.ean) ? filters.ean : [filters.ean], @@ -76,6 +78,14 @@ export async function getAssets({ }); } + if (filters.kindLike) { + must.push({ + wildcard: { + ['asset.kind']: filters.kindLike, + }, + }); + } + if (filters.eanLike) { must.push({ wildcard: { @@ -113,7 +123,7 @@ export async function getAssets({ }, }; - debug('Performing Asset Query', '\n\n', JSON.stringify(dsl, null, 2)); + debug('Performing Get Assets Query', '\n\n', JSON.stringify(dsl, null, 2)); const response = await esClient.search(dsl); return response.hits.hits.map((hit) => hit._source).filter((asset): asset is Asset => !!asset); diff --git a/x-pack/plugins/asset_manager/server/lib/get_related_assets.ts b/x-pack/plugins/asset_manager/server/lib/get_indirectly_related_assets.ts similarity index 84% rename from x-pack/plugins/asset_manager/server/lib/get_related_assets.ts rename to x-pack/plugins/asset_manager/server/lib/get_indirectly_related_assets.ts index 3e65ed565e5ac..fa9f3279ec497 100644 --- a/x-pack/plugins/asset_manager/server/lib/get_related_assets.ts +++ b/x-pack/plugins/asset_manager/server/lib/get_indirectly_related_assets.ts @@ -7,9 +7,10 @@ import { QueryDslQueryContainer, SearchRequest } from '@elastic/elasticsearch/lib/api/types'; import { debug } from '../../common/debug_log'; -import { Asset, AssetType, Relation, RelationField } from '../../common/types_api'; +import { Asset, AssetKind, Relation, RelationField } from '../../common/types_api'; import { ASSETS_INDEX_PREFIX } from '../constants'; import { ElasticsearchAccessorOptions } from '../types'; +import { isStringOrNonEmptyArray } from './utils'; interface GetRelatedAssetsOptions extends ElasticsearchAccessorOptions { size?: number; @@ -18,10 +19,10 @@ interface GetRelatedAssetsOptions extends ElasticsearchAccessorOptions { from?: string; to?: string; relation: Relation; - type?: AssetType[]; + kind?: AssetKind | AssetKind[]; } -export async function getRelatedAssets({ +export async function getIndirectlyRelatedAssets({ esClient, size = 100, from = 'now-24h', @@ -29,7 +30,7 @@ export async function getRelatedAssets({ ean, excludeEans, relation, - type, + kind, }: GetRelatedAssetsOptions): Promise { const relationField = relationToIndirectField(relation); const must: QueryDslQueryContainer[] = [ @@ -40,10 +41,10 @@ export async function getRelatedAssets({ }, ]; - if (type?.length) { + if (isStringOrNonEmptyArray(kind)) { must.push({ terms: { - ['asset.type']: type, + ['asset.kind']: Array.isArray(kind) ? kind : [kind], }, }); } @@ -88,7 +89,7 @@ export async function getRelatedAssets({ }, }; - debug('Performing Asset Query', '\n\n', JSON.stringify(dsl, null, 2)); + debug('Performing Indirectly Related Asset Query', '\n\n', JSON.stringify(dsl, null, 2)); const response = await esClient.search(dsl); return response.hits.hits.map((hit) => hit._source).filter((asset): asset is Asset => !!asset); diff --git a/x-pack/plugins/asset_manager/server/lib/sample_assets.ts b/x-pack/plugins/asset_manager/server/lib/sample_assets.ts index 762438b9d1b4b..2e9fbc2bcd5be 100644 --- a/x-pack/plugins/asset_manager/server/lib/sample_assets.ts +++ b/x-pack/plugins/asset_manager/server/lib/sample_assets.ts @@ -33,7 +33,7 @@ const sampleK8sClusters: AssetWithoutTimestamp[] = [ 'asset.kind': 'cluster', 'asset.id': 'cluster-001', 'asset.name': 'Cluster 001 (AWS EKS)', - 'asset.ean': 'k8s.cluster:cluster-001', + 'asset.ean': 'cluster:cluster-001', 'orchestrator.type': 'kubernetes', 'orchestrator.cluster.name': 'Cluster 001 (AWS EKS)', 'orchestrator.cluster.id': 'cluster-001', @@ -46,7 +46,7 @@ const sampleK8sClusters: AssetWithoutTimestamp[] = [ 'asset.kind': 'cluster', 'asset.id': 'cluster-002', 'asset.name': 'Cluster 002 (Azure AKS)', - 'asset.ean': 'k8s.cluster:cluster-002', + 'asset.ean': 'cluster:cluster-002', 'orchestrator.type': 'kubernetes', 'orchestrator.cluster.name': 'Cluster 002 (Azure AKS)', 'orchestrator.cluster.id': 'cluster-002', @@ -62,8 +62,8 @@ const sampleK8sNodes: AssetWithoutTimestamp[] = [ 'asset.kind': 'host', 'asset.id': 'node-101', 'asset.name': 'k8s-node-101-aws', - 'asset.ean': 'k8s.node:node-101', - 'asset.parents': ['k8s.cluster:cluster-001'], + 'asset.ean': 'host:node-101', + 'asset.parents': ['cluster:cluster-001'], 'orchestrator.type': 'kubernetes', 'orchestrator.cluster.name': 'Cluster 001 (AWS EKS)', 'orchestrator.cluster.id': 'cluster-001', @@ -76,8 +76,8 @@ const sampleK8sNodes: AssetWithoutTimestamp[] = [ 'asset.kind': 'host', 'asset.id': 'node-102', 'asset.name': 'k8s-node-102-aws', - 'asset.ean': 'k8s.node:node-102', - 'asset.parents': ['k8s.cluster:cluster-001'], + 'asset.ean': 'host:node-102', + 'asset.parents': ['cluster:cluster-001'], 'orchestrator.type': 'kubernetes', 'orchestrator.cluster.name': 'Cluster 001 (AWS EKS)', 'orchestrator.cluster.id': 'cluster-001', @@ -90,8 +90,8 @@ const sampleK8sNodes: AssetWithoutTimestamp[] = [ 'asset.kind': 'host', 'asset.id': 'node-103', 'asset.name': 'k8s-node-103-aws', - 'asset.ean': 'k8s.node:node-103', - 'asset.parents': ['k8s.cluster:cluster-001'], + 'asset.ean': 'host:node-103', + 'asset.parents': ['cluster:cluster-001'], 'orchestrator.type': 'kubernetes', 'orchestrator.cluster.name': 'Cluster 001 (AWS EKS)', 'orchestrator.cluster.id': 'cluster-001', @@ -107,73 +107,73 @@ const sampleK8sPods: AssetWithoutTimestamp[] = [ 'asset.kind': 'pod', 'asset.id': 'pod-200xrg1', 'asset.name': 'k8s-pod-200xrg1-aws', - 'asset.ean': 'k8s.pod:pod-200xrg1', - 'asset.parents': ['k8s.node:node-101'], - 'asset.references': ['k8s.cluster:cluster-001'], + 'asset.ean': 'pod:pod-200xrg1', + 'asset.parents': ['host:node-101'], + 'asset.references': ['cluster:cluster-001'], }, { 'asset.type': 'k8s.pod', 'asset.kind': 'pod', 'asset.id': 'pod-200dfp2', 'asset.name': 'k8s-pod-200dfp2-aws', - 'asset.ean': 'k8s.pod:pod-200dfp2', - 'asset.parents': ['k8s.node:node-101'], + 'asset.ean': 'pod:pod-200dfp2', + 'asset.parents': ['host:node-101'], }, { 'asset.type': 'k8s.pod', 'asset.kind': 'pod', 'asset.id': 'pod-200wwc3', 'asset.name': 'k8s-pod-200wwc3-aws', - 'asset.ean': 'k8s.pod:pod-200wwc3', - 'asset.parents': ['k8s.node:node-101'], + 'asset.ean': 'pod:pod-200wwc3', + 'asset.parents': ['host:node-101'], }, { 'asset.type': 'k8s.pod', 'asset.kind': 'pod', 'asset.id': 'pod-200naq4', 'asset.name': 'k8s-pod-200naq4-aws', - 'asset.ean': 'k8s.pod:pod-200naq4', - 'asset.parents': ['k8s.node:node-102'], + 'asset.ean': 'pod:pod-200naq4', + 'asset.parents': ['host:node-102'], }, { 'asset.type': 'k8s.pod', 'asset.kind': 'pod', 'asset.id': 'pod-200ohr5', 'asset.name': 'k8s-pod-200ohr5-aws', - 'asset.ean': 'k8s.pod:pod-200ohr5', - 'asset.parents': ['k8s.node:node-102'], + 'asset.ean': 'pod:pod-200ohr5', + 'asset.parents': ['host:node-102'], }, { 'asset.type': 'k8s.pod', 'asset.kind': 'pod', 'asset.id': 'pod-200yyx6', 'asset.name': 'k8s-pod-200yyx6-aws', - 'asset.ean': 'k8s.pod:pod-200yyx6', - 'asset.parents': ['k8s.node:node-103'], + 'asset.ean': 'pod:pod-200yyx6', + 'asset.parents': ['host:node-103'], }, { 'asset.type': 'k8s.pod', 'asset.kind': 'pod', 'asset.id': 'pod-200psd7', 'asset.name': 'k8s-pod-200psd7-aws', - 'asset.ean': 'k8s.pod:pod-200psd7', - 'asset.parents': ['k8s.node:node-103'], + 'asset.ean': 'pod:pod-200psd7', + 'asset.parents': ['host:node-103'], }, { 'asset.type': 'k8s.pod', 'asset.kind': 'pod', 'asset.id': 'pod-200wmc8', 'asset.name': 'k8s-pod-200wmc8-aws', - 'asset.ean': 'k8s.pod:pod-200wmc8', - 'asset.parents': ['k8s.node:node-103'], + 'asset.ean': 'pod:pod-200wmc8', + 'asset.parents': ['host:node-103'], }, { 'asset.type': 'k8s.pod', 'asset.kind': 'pod', 'asset.id': 'pod-200ugg9', 'asset.name': 'k8s-pod-200ugg9-aws', - 'asset.ean': 'k8s.pod:pod-200ugg9', - 'asset.parents': ['k8s.node:node-103'], + 'asset.ean': 'pod:pod-200ugg9', + 'asset.parents': ['host:node-103'], }, ]; @@ -183,30 +183,30 @@ const sampleCircularReferences: AssetWithoutTimestamp[] = [ 'asset.kind': 'host', 'asset.id': 'node-203', 'asset.name': 'k8s-node-203-aws', - 'asset.ean': 'k8s.node:node-203', + 'asset.ean': 'host:node-203', 'orchestrator.type': 'kubernetes', 'orchestrator.cluster.name': 'Cluster 001 (AWS EKS)', 'orchestrator.cluster.id': 'cluster-001', 'cloud.provider': 'aws', 'cloud.region': 'us-east-1', 'cloud.service.name': 'eks', - 'asset.references': ['k8s.pod:pod-203ugg9', 'k8s.pod:pod-203ugg5'], + 'asset.references': ['pod:pod-203ugg9', 'pod:pod-203ugg5'], }, { 'asset.type': 'k8s.pod', 'asset.kind': 'pod', 'asset.id': 'pod-203ugg5', 'asset.name': 'k8s-pod-203ugg5-aws', - 'asset.ean': 'k8s.pod:pod-203ugg5', - 'asset.references': ['k8s.node:node-203'], + 'asset.ean': 'pod:pod-203ugg5', + 'asset.references': ['host:node-203'], }, { 'asset.type': 'k8s.pod', 'asset.kind': 'pod', 'asset.id': 'pod-203ugg9', 'asset.name': 'k8s-pod-203ugg9-aws', - 'asset.ean': 'k8s.pod:pod-203ugg9', - 'asset.references': ['k8s.node:node-203'], + 'asset.ean': 'pod:pod-203ugg9', + 'asset.references': ['host:node-203'], }, ]; diff --git a/x-pack/plugins/asset_manager/server/lib/utils.ts b/x-pack/plugins/asset_manager/server/lib/utils.ts index 9db0d573a6221..fa9cf965771ef 100644 --- a/x-pack/plugins/asset_manager/server/lib/utils.ts +++ b/x-pack/plugins/asset_manager/server/lib/utils.ts @@ -23,3 +23,15 @@ export const isValidRange = (from: string, to: string): boolean => { } return true; }; + +export function isStringOrNonEmptyArray( + value: string | string[] | undefined +): value is string | string[] { + if (typeof value === 'undefined') { + return false; + } + if (Array.isArray(value) && value.length === 0) { + return false; + } + return true; +} diff --git a/x-pack/plugins/asset_manager/server/routes/assets.ts b/x-pack/plugins/asset_manager/server/routes/assets.ts index 5c554f2454fd4..55d45c16db7bb 100644 --- a/x-pack/plugins/asset_manager/server/routes/assets.ts +++ b/x-pack/plugins/asset_manager/server/routes/assets.ts @@ -16,14 +16,18 @@ import { createLiteralValueFromUndefinedRT, } from '@kbn/io-ts-utils'; import { debug } from '../../common/debug_log'; -import { AssetType, assetTypeRT, relationRT } from '../../common/types_api'; +import { assetTypeRT, assetKindRT, relationRT } from '../../common/types_api'; import { ASSET_MANAGER_API_BASE } from '../constants'; import { getAssets } from '../lib/get_assets'; import { getAllRelatedAssets } from '../lib/get_all_related_assets'; import { SetupRouteOptions } from './types'; import { getEsClientFromContext } from './utils'; import { AssetNotFoundError } from '../lib/errors'; -import { isValidRange, toArray } from '../lib/utils'; +import { isValidRange } from '../lib/utils'; + +function maybeArrayRT(t: rt.Mixed) { + return rt.union([rt.array(t), t]); +} const sizeRT = rt.union([inRangeFromStringRt(1, 100), createLiteralValueFromUndefinedRT(10)]); const assetDateRT = rt.union([dateRt, datemathStringRt]); @@ -31,8 +35,9 @@ const getAssetsQueryOptionsRT = rt.exact( rt.partial({ from: assetDateRT, to: assetDateRT, - type: rt.union([rt.array(assetTypeRT), assetTypeRT]), - ean: rt.union([rt.array(rt.string), rt.string]), + type: maybeArrayRT(assetTypeRT), + kind: maybeArrayRT(assetKindRT), + ean: maybeArrayRT(rt.string), size: sizeRT, }) ); @@ -46,7 +51,8 @@ const getAssetsDiffQueryOptionsRT = rt.exact( bTo: assetDateRT, }), rt.partial({ - type: rt.union([rt.array(assetTypeRT), assetTypeRT]), + type: maybeArrayRT(assetTypeRT), + kind: maybeArrayRT(assetKindRT), }), ]) ); @@ -62,7 +68,8 @@ const getRelatedAssetsQueryOptionsRT = rt.exact( }), rt.partial({ to: assetDateRT, - type: rt.union([rt.array(assetTypeRT), assetTypeRT]), + type: maybeArrayRT(assetTypeRT), + kind: maybeArrayRT(assetKindRT), }), ]) ); @@ -89,6 +96,12 @@ export function assetsRoutes({ router }: SetupR }); } + if (filters.kind && filters.ean) { + return res.badRequest({ + body: 'Filters "kind" and "ean" are mutually exclusive but found both.', + }); + } + const esClient = await getEsClientFromContext(context); try { @@ -96,7 +109,10 @@ export function assetsRoutes({ router }: SetupR return res.ok({ body: { results } }); } catch (error: unknown) { debug('error looking up asset records', error); - return res.customError({ statusCode: 500 }); + return res.customError({ + statusCode: 500, + body: { message: 'Error while looking up asset records - ' + `${error}` }, + }); } } ); @@ -112,11 +128,9 @@ export function assetsRoutes({ router }: SetupR async (context, req, res) => { // Add references into sample data and write integration tests - const { from, to, ean, relation, maxDistance, size } = req.query || {}; + const { from, to, ean, relation, maxDistance, size, type, kind } = req.query || {}; const esClient = await getEsClientFromContext(context); - const type = toArray(req.query.type); - if (to && !isValidRange(from, to)) { return res.badRequest({ body: `Time range cannot move backwards in time. "to" (${to}) is before "from" (${from}).`, @@ -131,6 +145,7 @@ export function assetsRoutes({ router }: SetupR from, to, type, + kind, maxDistance, size, relation, @@ -156,8 +171,9 @@ export function assetsRoutes({ router }: SetupR }, }, async (context, req, res) => { - const { aFrom, aTo, bFrom, bTo } = req.query; - const type = toArray(req.query.type); + const { aFrom, aTo, bFrom, bTo, type, kind } = req.query; + // const type = toArray(req.query.type); + // const kind = toArray(req.query.kind); if (!isValidRange(aFrom, aTo)) { return res.badRequest({ @@ -180,6 +196,7 @@ export function assetsRoutes({ router }: SetupR from: aFrom, to: aTo, type, + kind, }, }); @@ -189,6 +206,7 @@ export function assetsRoutes({ router }: SetupR from: bFrom, to: bTo, type, + kind, }, }); diff --git a/x-pack/plugins/asset_manager/server/routes/sample_assets.ts b/x-pack/plugins/asset_manager/server/routes/sample_assets.ts index 64c975d46a812..98f7f32051f3f 100644 --- a/x-pack/plugins/asset_manager/server/routes/sample_assets.ts +++ b/x-pack/plugins/asset_manager/server/routes/sample_assets.ts @@ -103,38 +103,38 @@ export function sampleAssetsRoutes({ async (context, req, res) => { const esClient = await getEsClientFromContext(context); - const sampleDataIndices = await esClient.indices.get({ - index: 'assets-*-sample_data', + const sampleDataStreams = await esClient.indices.getDataStream({ + name: 'assets-*-sample_data', expand_wildcards: 'all', }); - const deletedIndices: string[] = []; + const deletedDataStreams: string[] = []; let errorWhileDeleting: string | null = null; - const indicesToDelete = Object.keys(sampleDataIndices); + const dataStreamsToDelete = sampleDataStreams.data_streams.map((ds) => ds.name); - for (let i = 0; i < indicesToDelete.length; i++) { - const index = indicesToDelete[i]; + for (let i = 0; i < dataStreamsToDelete.length; i++) { + const dsName = dataStreamsToDelete[i]; try { - await esClient.indices.delete({ index }); - deletedIndices.push(index); + await esClient.indices.deleteDataStream({ name: dsName }); + deletedDataStreams.push(dsName); } catch (error: any) { errorWhileDeleting = typeof error.message === 'string' ? error.message - : `Unknown error occurred while deleting indices, at index ${index}`; + : `Unknown error occurred while deleting sample data streams, at data stream name: ${dsName}`; break; } } - if (deletedIndices.length === indicesToDelete.length) { - return res.ok({ body: { deleted: deletedIndices } }); + if (!errorWhileDeleting && deletedDataStreams.length === dataStreamsToDelete.length) { + return res.ok({ body: { deleted: deletedDataStreams } }); } else { return res.custom({ statusCode: 500, body: { - message: ['Not all matching indices were deleted', errorWhileDeleting].join(' - '), - deleted: deletedIndices, - matching: indicesToDelete, + message: ['Not all found data streams were deleted', errorWhileDeleting].join(' - '), + deleted: deletedDataStreams, + matching: dataStreamsToDelete, }, }); } diff --git a/x-pack/test/api_integration/apis/asset_manager/tests/assets.ts b/x-pack/test/api_integration/apis/asset_manager/tests/assets.ts index a78c7e59765ad..40d1a91b5edbe 100644 --- a/x-pack/test/api_integration/apis/asset_manager/tests/assets.ts +++ b/x-pack/test/api_integration/apis/asset_manager/tests/assets.ts @@ -136,6 +136,72 @@ export default function ({ getService }: FtrProviderContext) { ); }); + it('should return assets filtered by a single asset.kind value', async () => { + await createSampleAssets(supertest); + + const singleSampleKind = sampleAssetDocs[0]['asset.kind']; + const samplesForKind = sampleAssetDocs.filter( + (doc) => doc['asset.kind'] === singleSampleKind + ); + + const getResponse = await supertest + .get(ASSETS_ENDPOINT) + .query({ size: sampleAssetDocs.length, from: 'now-1d', kind: singleSampleKind }) + .expect(200); + + expect(getResponse.body).to.have.property('results'); + expect(getResponse.body.results.length).to.equal(samplesForKind.length); + }); + + it('should return assets filtered by multiple asset.kind values (OR)', async () => { + await createSampleAssets(supertest); + + // Dynamically grab all asset.kind values from the sample asset data set + const sampleKindSet: Set = new Set(); + for (let i = 0; i < sampleAssetDocs.length; i++) { + sampleKindSet.add(sampleAssetDocs[i]['asset.kind']!); + } + const sampleKinds = Array.from(sampleKindSet); + if (sampleKinds.length <= 2) { + throw new Error( + 'Not enough asset kind values in sample asset documents, need more than two to test filtering by multiple kinds' + ); + } + + // Pick the first two unique kinds from the sample data set + const filterByKinds = sampleKinds.slice(0, 2); + + // Track a reference to how many docs should be returned for these two kinds + const samplesForFilteredKinds = sampleAssetDocs.filter((doc) => + filterByKinds.includes(doc['asset.kind']!) + ); + + expect(samplesForFilteredKinds.length).to.be.lessThan(sampleAssetDocs.length); + + // Request assets for multiple types (with a size matching the number of total sample asset docs) + const getResponse = await supertest + .get(ASSETS_ENDPOINT) + .query({ size: sampleAssetDocs.length, from: 'now-1d', kind: filterByKinds }) + .expect(200); + + expect(getResponse.body).to.have.property('results'); + expect(getResponse.body.results.length).to.equal(samplesForFilteredKinds.length); + }); + + it('should reject requests that try to filter by both kind and ean', async () => { + const sampleKind = sampleAssetDocs[0]['asset.kind']; + const sampleEan = sampleAssetDocs[0]['asset.ean']; + + const getResponse = await supertest + .get(ASSETS_ENDPOINT) + .query({ kind: sampleKind, ean: sampleEan }) + .expect(400); + + expect(getResponse.body.message).to.equal( + 'Filters "kind" and "ean" are mutually exclusive but found both.' + ); + }); + it('should return the asset matching a single ean', async () => { await createSampleAssets(supertest); diff --git a/x-pack/test/api_integration/apis/asset_manager/tests/assets_diff.ts b/x-pack/test/api_integration/apis/asset_manager/tests/assets_diff.ts index 8ae204989b910..33b5a465842dd 100644 --- a/x-pack/test/api_integration/apis/asset_manager/tests/assets_diff.ts +++ b/x-pack/test/api_integration/apis/asset_manager/tests/assets_diff.ts @@ -58,7 +58,7 @@ export default function ({ getService }: FtrProviderContext) { '[request query]: Failed to validate: \n in /0/bTo: undefined does not match expected type Date\n in /0/bTo: undefined does not match expected type datemath' ); - await supertest + return await supertest .get(DIFF_ENDPOINT) .query({ aFrom: timestamp, aTo: timestamp, bFrom: timestamp, bTo: timestamp }) .expect(200); @@ -95,7 +95,7 @@ export default function ({ getService }: FtrProviderContext) { `Time range cannot move backwards in time. "bTo" (${oneHourAgo}) is before "bFrom" (${isoNow}).` ); - await supertest + return await supertest .get(DIFF_ENDPOINT) .query({ aFrom: oneHourAgo, @@ -107,20 +107,26 @@ export default function ({ getService }: FtrProviderContext) { }); it('should return the difference in assets present between two time ranges', async () => { - const onlyInA = sampleAssetDocs.slice(0, 2); - const onlyInB = sampleAssetDocs.slice(sampleAssetDocs.length - 2); - const inBoth = sampleAssetDocs.slice(2, sampleAssetDocs.length - 2); + const onlyInA = sampleAssetDocs.slice(0, 2); // first two sample assets + const onlyInB = sampleAssetDocs.slice(sampleAssetDocs.length - 2); // last two sample assets + const inBoth = sampleAssetDocs.slice(2, sampleAssetDocs.length - 2); // everything between first 2, last 2 const now = new Date(); const oneHourAgo = new Date(now.getTime() - 1000 * 60 * 60 * 1); const twoHoursAgo = new Date(now.getTime() - 1000 * 60 * 60 * 2); + + // Set 1: Two hours ago, excludes inBoth + onlyInB (leaves just onlyInA) await createSampleAssets(supertest, { baseDateTime: twoHoursAgo.toISOString(), excludeEans: inBoth.concat(onlyInB).map((asset) => asset['asset.ean']), }); + + // Set 2: One hour ago, excludes onlyInA + onlyInB (leaves just inBoth) await createSampleAssets(supertest, { baseDateTime: oneHourAgo.toISOString(), excludeEans: onlyInA.concat(onlyInB).map((asset) => asset['asset.ean']), }); + + // Set 3: Right now, excludes inBoth + onlyInA (leaves just onlyInB) await createSampleAssets(supertest, { excludeEans: inBoth.concat(onlyInA).map((asset) => asset['asset.ean']), }); @@ -130,6 +136,8 @@ export default function ({ getService }: FtrProviderContext) { const seventyMinuesAgo = new Date(now.getTime() - 1000 * 60 * 70 * 1); const tenMinutesAfterNow = new Date(now.getTime() + 1000 * 60 * 10); + // Range A: 2h10m ago - 50m ago (Sets 1 and 2) + // Range B: 70m ago - 10m after now (Sets 2 and 3) const getResponse = await supertest .get(DIFF_ENDPOINT) .query({ diff --git a/x-pack/test/api_integration/apis/asset_manager/tests/assets_related.ts b/x-pack/test/api_integration/apis/asset_manager/tests/assets_related.ts index edabcb734893b..3cf8310edd4aa 100644 --- a/x-pack/test/api_integration/apis/asset_manager/tests/assets_related.ts +++ b/x-pack/test/api_integration/apis/asset_manager/tests/assets_related.ts @@ -40,18 +40,18 @@ export default function ({ getService }: FtrProviderContext) { const relations = [ { name: 'ancestors', - ean: 'k8s.node:node-101', - expectedRelatedEans: ['k8s.cluster:cluster-001'], + ean: 'host:node-101', + expectedRelatedEans: ['cluster:cluster-001'], }, { name: 'descendants', - ean: 'k8s.cluster:cluster-001', - expectedRelatedEans: ['k8s.node:node-101', 'k8s.node:node-102', 'k8s.node:node-103'], + ean: 'cluster:cluster-001', + expectedRelatedEans: ['host:node-101', 'host:node-102', 'host:node-103'], }, { name: 'references', - ean: 'k8s.pod:pod-200xrg1', - expectedRelatedEans: ['k8s.cluster:cluster-001'], + ean: 'pod:pod-200xrg1', + expectedRelatedEans: ['cluster:cluster-001'], }, ]; @@ -158,8 +158,8 @@ export default function ({ getService }: FtrProviderContext) { expect( results.references.map((asset: Asset) => pick(asset, ['asset.ean', 'distance'])) ).to.eql([ - { 'asset.ean': 'k8s.node:node-203', distance: 1 }, - { 'asset.ean': 'k8s.pod:pod-203ugg9', distance: 2 }, + { 'asset.ean': 'host:node-203', distance: 1 }, + { 'asset.ean': 'pod:pod-203ugg9', distance: 2 }, ]); }); @@ -317,8 +317,8 @@ export default function ({ getService }: FtrProviderContext) { }); }); - describe('with asset.type filters', () => { - it('should filter by the provided asset type', async () => { + describe('with asset.kind filters', () => { + it('should filter by the provided asset kind', async () => { await createSampleAssets(supertest); const sampleCluster = sampleAssetDocs.find( @@ -333,7 +333,7 @@ export default function ({ getService }: FtrProviderContext) { from: 'now-1d', ean: sampleCluster!['asset.ean'], maxDistance: 1, - type: ['k8s.pod'], + kind: ['pod'], }) .expect(200); @@ -358,7 +358,7 @@ export default function ({ getService }: FtrProviderContext) { from: 'now-1d', ean: sampleCluster!['asset.ean'], maxDistance: 2, - type: ['k8s.pod'], + kind: ['pod'], }) .expect(200); @@ -367,7 +367,7 @@ export default function ({ getService }: FtrProviderContext) { } = getResponse; expect(results.descendants).to.have.length(9); expect(results.descendants.every((asset: { distance: number }) => asset.distance === 2)); - expect(results.descendants.every((asset: Asset) => asset['asset.type'] === 'k8s.pod')); + expect(results.descendants.every((asset: Asset) => asset['asset.kind'] === 'pod')); }); }); });