diff --git a/x-pack/plugins/asset_manager/common/types_api.ts b/x-pack/plugins/asset_manager/common/types_api.ts index e1eb6e61f83c3..a72392316565e 100644 --- a/x-pack/plugins/asset_manager/common/types_api.ts +++ b/x-pack/plugins/asset_manager/common/types_api.ts @@ -15,7 +15,7 @@ export const assetTypeRT = rt.union([ export type AssetType = rt.TypeOf; -export type AssetKind = 'unknown' | 'node'; +export type AssetKind = 'cluster' | 'host' | 'pod' | 'container' | 'service'; export type AssetStatus = | 'CREATING' | 'ACTIVE' @@ -47,17 +47,20 @@ export interface ECSDocument extends WithTimestamp { 'orchestrator.cluster.version'?: string; 'cloud.provider'?: CloudProviderName; + 'cloud.instance.id'?: string; 'cloud.region'?: string; 'cloud.service.name'?: string; + + 'service.environment'?: string; } export interface Asset extends ECSDocument { 'asset.collection_version'?: string; 'asset.ean': string; 'asset.id': string; - 'asset.kind'?: AssetKind; + 'asset.kind': AssetKind; 'asset.name'?: string; - 'asset.type': AssetType; + 'asset.type'?: AssetType; 'asset.status'?: AssetStatus; 'asset.parents'?: string | string[]; 'asset.children'?: string | string[]; diff --git a/x-pack/plugins/asset_manager/server/constants.ts b/x-pack/plugins/asset_manager/server/constants.ts index 0aa1cb467df48..8f6bbef10f258 100644 --- a/x-pack/plugins/asset_manager/server/constants.ts +++ b/x-pack/plugins/asset_manager/server/constants.ts @@ -7,3 +7,7 @@ export const ASSETS_INDEX_PREFIX = 'assets'; export const ASSET_MANAGER_API_BASE = '/api/asset-manager'; + +export const LOGS_INDICES = 'logs-*,filebeat-*'; +export const APM_INDICES = 'traces-*,apm*,metrics-apm*,logs-apm*'; +export const METRICS_INDICES = 'metrics-*,metricbeat-*'; diff --git a/x-pack/plugins/asset_manager/server/lib/get_all_related_assets.test.ts b/x-pack/plugins/asset_manager/server/lib/get_all_related_assets.test.ts index 7eaa05dbd35bb..6859bfec31910 100644 --- a/x-pack/plugins/asset_manager/server/lib/get_all_related_assets.test.ts +++ b/x-pack/plugins/asset_manager/server/lib/get_all_related_assets.test.ts @@ -27,6 +27,7 @@ describe('getAllRelatedAssets', () => { const primaryAsset: AssetWithoutTimestamp = { 'asset.ean': 'primary-which-does-not-exist', 'asset.type': 'k8s.pod', + 'asset.kind': 'pod', 'asset.id': uuid(), }; @@ -53,6 +54,7 @@ describe('getAllRelatedAssets', () => { const primaryAssetWithoutParents: AssetWithoutTimestamp = { 'asset.ean': 'primary-ean', 'asset.type': 'k8s.pod', + 'asset.kind': 'pod', 'asset.id': uuid(), 'asset.parents': [], }; @@ -82,12 +84,14 @@ describe('getAllRelatedAssets', () => { const parentAsset: AssetWithoutTimestamp = { 'asset.ean': 'primary-ean', 'asset.type': 'k8s.pod', + 'asset.kind': 'pod', 'asset.id': uuid(), }; const primaryAssetWithDirectParent: AssetWithoutTimestamp = { 'asset.ean': 'primary-ean', 'asset.type': 'k8s.pod', + 'asset.kind': 'pod', 'asset.id': uuid(), 'asset.parents': [parentAsset['asset.ean']], }; @@ -124,6 +128,7 @@ describe('getAllRelatedAssets', () => { const primaryAssetWithIndirectParent: AssetWithoutTimestamp = { 'asset.ean': 'primary-ean', 'asset.type': 'k8s.pod', + 'asset.kind': 'pod', 'asset.id': uuid(), 'asset.parents': [], }; @@ -131,6 +136,7 @@ describe('getAllRelatedAssets', () => { const parentAsset: AssetWithoutTimestamp = { 'asset.ean': 'primary-ean', 'asset.type': 'k8s.pod', + 'asset.kind': 'pod', 'asset.id': uuid(), 'asset.children': [primaryAssetWithIndirectParent['asset.ean']], }; @@ -167,6 +173,7 @@ describe('getAllRelatedAssets', () => { const directlyReferencedParent: AssetWithoutTimestamp = { 'asset.ean': 'directly-referenced-parent-ean', 'asset.type': 'k8s.pod', + 'asset.kind': 'pod', 'asset.id': uuid(), 'asset.children': [], }; @@ -174,6 +181,7 @@ describe('getAllRelatedAssets', () => { const primaryAsset: AssetWithoutTimestamp = { 'asset.ean': 'primary-ean', 'asset.type': 'k8s.pod', + 'asset.kind': 'pod', 'asset.id': uuid(), 'asset.parents': [directlyReferencedParent['asset.ean']], }; @@ -181,6 +189,7 @@ describe('getAllRelatedAssets', () => { const indirectlyReferencedParent: AssetWithoutTimestamp = { 'asset.ean': 'indirectly-referenced-parent-ean', 'asset.type': 'k8s.pod', + 'asset.kind': 'pod', 'asset.id': uuid(), 'asset.children': [primaryAsset['asset.ean']], }; @@ -221,12 +230,14 @@ describe('getAllRelatedAssets', () => { const parentAsset: AssetWithoutTimestamp = { 'asset.ean': 'parent-ean', 'asset.type': 'k8s.pod', + 'asset.kind': 'pod', 'asset.id': uuid(), }; const primaryAsset: AssetWithoutTimestamp = { 'asset.ean': 'primary-ean', 'asset.type': 'k8s.pod', + 'asset.kind': 'pod', 'asset.id': uuid(), }; @@ -266,12 +277,14 @@ describe('getAllRelatedAssets', () => { const distance6Parent: AssetWithoutTimestamp = { 'asset.ean': 'parent-5-ean', 'asset.type': 'k8s.pod', + 'asset.kind': 'pod', 'asset.id': uuid(), }; const distance5Parent: AssetWithoutTimestamp = { 'asset.ean': 'parent-5-ean', 'asset.type': 'k8s.pod', + 'asset.kind': 'pod', 'asset.id': uuid(), 'asset.parents': [distance6Parent['asset.ean']], }; @@ -279,6 +292,7 @@ describe('getAllRelatedAssets', () => { const distance4Parent: AssetWithoutTimestamp = { 'asset.ean': 'parent-4-ean', 'asset.type': 'k8s.pod', + 'asset.kind': 'pod', 'asset.id': uuid(), 'asset.parents': [distance5Parent['asset.ean']], }; @@ -286,6 +300,7 @@ describe('getAllRelatedAssets', () => { const distance3Parent: AssetWithoutTimestamp = { 'asset.ean': 'parent-3-ean', 'asset.type': 'k8s.pod', + 'asset.kind': 'pod', 'asset.id': uuid(), 'asset.parents': [distance4Parent['asset.ean']], }; @@ -293,6 +308,7 @@ describe('getAllRelatedAssets', () => { const distance2Parent: AssetWithoutTimestamp = { 'asset.ean': 'parent-2-ean', 'asset.type': 'k8s.pod', + 'asset.kind': 'pod', 'asset.id': uuid(), 'asset.parents': [distance3Parent['asset.ean']], }; @@ -300,6 +316,7 @@ describe('getAllRelatedAssets', () => { const distance1Parent: AssetWithoutTimestamp = { 'asset.ean': 'parent-1-ean', 'asset.type': 'k8s.pod', + 'asset.kind': 'pod', 'asset.id': uuid(), 'asset.parents': [distance2Parent['asset.ean']], }; @@ -307,6 +324,7 @@ describe('getAllRelatedAssets', () => { const primaryAsset: AssetWithoutTimestamp = { 'asset.ean': 'primary-ean', 'asset.type': 'k8s.pod', + 'asset.kind': 'pod', 'asset.id': uuid(), 'asset.parents': [distance1Parent['asset.ean']], }; @@ -368,12 +386,14 @@ describe('getAllRelatedAssets', () => { const distance3Parent: AssetWithoutTimestamp = { 'asset.ean': 'parent-3-ean', 'asset.type': 'k8s.pod', + 'asset.kind': 'pod', 'asset.id': uuid(), }; const distance2Parent: AssetWithoutTimestamp = { 'asset.ean': 'parent-2-ean', 'asset.type': 'k8s.pod', + 'asset.kind': 'pod', 'asset.id': uuid(), 'asset.parents': [distance3Parent['asset.ean']], }; @@ -381,6 +401,7 @@ describe('getAllRelatedAssets', () => { const distance1Parent: AssetWithoutTimestamp = { 'asset.ean': 'parent-1-ean', 'asset.type': 'k8s.pod', + 'asset.kind': 'pod', 'asset.id': uuid(), 'asset.parents': [distance2Parent['asset.ean']], }; @@ -388,6 +409,7 @@ describe('getAllRelatedAssets', () => { const primaryAsset: AssetWithoutTimestamp = { 'asset.ean': 'primary-ean', 'asset.type': 'k8s.pod', + 'asset.kind': 'pod', 'asset.id': uuid(), 'asset.parents': [distance1Parent['asset.ean']], }; @@ -435,30 +457,35 @@ describe('getAllRelatedAssets', () => { const distance2ParentA: AssetWithoutTimestamp = { 'asset.ean': 'parent-2-ean-a', 'asset.type': 'k8s.pod', + 'asset.kind': 'pod', 'asset.id': uuid(), }; const distance2ParentB: AssetWithoutTimestamp = { 'asset.ean': 'parent-2-ean-b', 'asset.type': 'k8s.pod', + 'asset.kind': 'pod', 'asset.id': uuid(), }; const distance2ParentC: AssetWithoutTimestamp = { 'asset.ean': 'parent-2-ean-c', 'asset.type': 'k8s.pod', + 'asset.kind': 'pod', 'asset.id': uuid(), }; const distance2ParentD: AssetWithoutTimestamp = { 'asset.ean': 'parent-2-ean-d', 'asset.type': 'k8s.pod', + 'asset.kind': 'pod', 'asset.id': uuid(), }; const distance1ParentA: AssetWithoutTimestamp = { 'asset.ean': 'parent-1-ean-a', 'asset.type': 'k8s.pod', + 'asset.kind': 'pod', 'asset.id': uuid(), 'asset.parents': [distance2ParentA['asset.ean'], distance2ParentB['asset.ean']], }; @@ -466,6 +493,7 @@ describe('getAllRelatedAssets', () => { const distance1ParentB: AssetWithoutTimestamp = { 'asset.ean': 'parent-1-ean-b', 'asset.type': 'k8s.pod', + 'asset.kind': 'pod', 'asset.id': uuid(), 'asset.parents': [distance2ParentC['asset.ean'], distance2ParentD['asset.ean']], }; @@ -473,6 +501,7 @@ describe('getAllRelatedAssets', () => { const primaryAsset: AssetWithoutTimestamp = { 'asset.ean': 'primary-ean', 'asset.type': 'k8s.pod', + 'asset.kind': 'pod', 'asset.id': uuid(), 'asset.parents': [distance1ParentA['asset.ean'], distance1ParentB['asset.ean']], }; diff --git a/x-pack/plugins/asset_manager/server/lib/get_all_related_assets.ts b/x-pack/plugins/asset_manager/server/lib/get_all_related_assets.ts index f297881e1aec6..9f2ab5f114f9e 100644 --- a/x-pack/plugins/asset_manager/server/lib/get_all_related_assets.ts +++ b/x-pack/plugins/asset_manager/server/lib/get_all_related_assets.ts @@ -67,7 +67,7 @@ export async function getAllRelatedAssets( return { primary, [relation]: type.length - ? relatedAssets.filter((asset) => type.includes(asset['asset.type'])) + ? relatedAssets.filter((asset) => asset['asset.type'] && type.includes(asset['asset.type'])) : relatedAssets, }; } diff --git a/x-pack/plugins/asset_manager/server/lib/implicit_collection/collector_runner.test.ts b/x-pack/plugins/asset_manager/server/lib/implicit_collection/collector_runner.test.ts new file mode 100644 index 0000000000000..da82896d4f9ae --- /dev/null +++ b/x-pack/plugins/asset_manager/server/lib/implicit_collection/collector_runner.test.ts @@ -0,0 +1,102 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { CollectorRunner } from './collector_runner'; +import { CollectorOptions } from './collectors'; +import { Asset } from '../../../common/types_api'; + +const getMockClient = () => ({ + bulk: jest.fn().mockResolvedValue({ errors: false }), +}); + +const getMockLogger = () => + ({ + info: jest.fn(), + error: jest.fn(), + } as unknown as Logger); + +describe(__filename, () => { + it('runs registered collectors', async () => { + const runner = new CollectorRunner({ + inputClient: getMockClient() as unknown as ElasticsearchClient, + outputClient: getMockClient() as unknown as ElasticsearchClient, + logger: getMockLogger(), + intervalMs: 1, + }); + + const collector1 = jest.fn(async (opts: CollectorOptions) => { + return []; + }); + const collector2 = jest.fn(async (opts: CollectorOptions) => { + return []; + }); + + runner.registerCollector('foo', collector1); + runner.registerCollector('foo', collector2); + + await runner.run(); + + expect(collector1.mock.calls).to.have.length(1); + expect(collector2.mock.calls).to.have.length(1); + }); + + it('is resilient to failing collectors', async () => { + const runner = new CollectorRunner({ + outputClient: getMockClient() as unknown as ElasticsearchClient, + inputClient: getMockClient() as unknown as ElasticsearchClient, + logger: getMockLogger(), + intervalMs: 1, + }); + + const collector1 = jest.fn(async (opts: CollectorOptions) => { + throw new Error('no'); + }); + const collector2 = jest.fn(async (opts: CollectorOptions) => { + return []; + }); + + runner.registerCollector('foo', collector1); + runner.registerCollector('foo', collector2); + + await runner.run(); + + expect(collector1.mock.calls).to.have.length(1); + expect(collector2.mock.calls).to.have.length(1); + }); + + it('stores collectors results in elasticsearch', async () => { + const outputClient = getMockClient(); + const runner = new CollectorRunner({ + outputClient: outputClient as unknown as ElasticsearchClient, + inputClient: getMockClient() as unknown as ElasticsearchClient, + logger: getMockLogger(), + intervalMs: 1, + }); + + const collector = jest.fn(async (opts: CollectorOptions) => { + return [ + { 'asset.kind': 'container', 'asset.ean': 'foo' }, + { 'asset.kind': 'pod', 'asset.ean': 'bar' }, + ] as Asset[]; + }); + + runner.registerCollector('foo', collector); + + await runner.run(); + + expect(outputClient.bulk.mock.calls[0][0]).to.eql({ + body: [ + { create: { _index: 'assets-container-default' } }, + { 'asset.kind': 'container', 'asset.ean': 'foo' }, + { create: { _index: 'assets-pod-default' } }, + { 'asset.kind': 'pod', 'asset.ean': 'bar' }, + ], + }); + }); +}); diff --git a/x-pack/plugins/asset_manager/server/lib/implicit_collection/collector_runner.ts b/x-pack/plugins/asset_manager/server/lib/implicit_collection/collector_runner.ts new file mode 100644 index 0000000000000..eb60a49e0240d --- /dev/null +++ b/x-pack/plugins/asset_manager/server/lib/implicit_collection/collector_runner.ts @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ImplicitCollectionOptions } from '.'; +import { Collector } from './collectors'; +import { Asset } from '../../../common/types_api'; + +export class CollectorRunner { + private collectors: Array<{ name: string; collector: Collector }> = []; + + constructor(private options: ImplicitCollectionOptions) {} + + registerCollector(name: string, collector: Collector) { + this.collectors.push({ name, collector }); + } + + async run() { + const collectorOptions = { + client: this.options.inputClient, + from: Date.now() - this.options.intervalMs, + }; + + for (let i = 0; i < this.collectors.length; i++) { + const { name, collector } = this.collectors[i]; + this.options.logger.info(`Collector '${name}' started`); + + const assets = await collector(collectorOptions) + .then((collectedAssets) => { + this.options.logger.info(`Collector '${name}' found ${collectedAssets.length} assets`); + return collectedAssets; + }) + .catch((err) => { + this.options.logger.error(`Collector '${name}' execution failure: ${err}`); + return []; + }); + + if (assets.length) { + const bulkBody = assets.flatMap((asset: Asset) => { + return [{ create: { _index: `assets-${asset['asset.kind']}-default` } }, asset]; + }); + + await this.options.outputClient + .bulk({ body: bulkBody }) + .then((res) => { + if (res.errors) { + this.options.logger.error( + `Failure writing assets documents from collector '${name}': ${JSON.stringify(res)}` + ); + } + }) + .catch((err) => { + this.options.logger.error( + `Failure writing assets documents from collector '${name}': ${err}` + ); + }); + } + } + } +} diff --git a/x-pack/plugins/asset_manager/server/lib/implicit_collection/collectors/containers.ts b/x-pack/plugins/asset_manager/server/lib/implicit_collection/collectors/containers.ts new file mode 100644 index 0000000000000..845e81d5c5640 --- /dev/null +++ b/x-pack/plugins/asset_manager/server/lib/implicit_collection/collectors/containers.ts @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { APM_INDICES, LOGS_INDICES, METRICS_INDICES } from '../../../constants'; +import { Asset } from '../../../../common/types_api'; +import { CollectorOptions } from '.'; + +export async function collectContainers({ client, from }: CollectorOptions) { + const dsl = { + index: [APM_INDICES, LOGS_INDICES, METRICS_INDICES], + size: 1000, + collapse: { + field: 'container.id', + }, + sort: [{ _score: 'desc' }, { '@timestamp': 'desc' }], + _source: false, + fields: [ + 'kubernetes.*', + 'cloud.provider', + 'orchestrator.cluster.name', + 'host.name', + 'host.hostname', + ], + query: { + bool: { + filter: [ + { + range: { + '@timestamp': { + gte: from, + }, + }, + }, + ], + should: [ + { exists: { field: 'kubernetes.container.id' } }, + { exists: { field: 'kubernetes.pod.uid' } }, + { exists: { field: 'kubernetes.node.name' } }, + { exists: { field: 'host.hostname' } }, + ], + }, + }, + }; + + const esResponse = await client.search(dsl); + + const containers = esResponse.hits.hits.reduce((acc: Asset[], hit: any) => { + const { fields = {} } = hit; + const containerId = fields['container.id']; + const podUid = fields['kubernetes.pod.uid']; + const nodeName = fields['kubernetes.node.name']; + + const parentEan = podUid ? `pod:${podUid}` : `host:${fields['host.hostname']}`; + + const container: Asset = { + '@timestamp': new Date().toISOString(), + 'asset.kind': 'container', + 'asset.id': containerId, + 'asset.ean': `container:${containerId}`, + 'asset.parents': [parentEan], + }; + + if (nodeName) { + container['asset.references'] = [`host:${nodeName}`]; + } + + acc.push(container); + + return acc; + }, []); + + return containers; +} diff --git a/x-pack/plugins/asset_manager/server/lib/implicit_collection/collectors/hosts.ts b/x-pack/plugins/asset_manager/server/lib/implicit_collection/collectors/hosts.ts new file mode 100644 index 0000000000000..f5409495c3fad --- /dev/null +++ b/x-pack/plugins/asset_manager/server/lib/implicit_collection/collectors/hosts.ts @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { APM_INDICES, METRICS_INDICES, LOGS_INDICES } from '../../../constants'; +import { Asset } from '../../../../common/types_api'; +import { CollectorOptions } from '.'; + +export async function collectHosts({ client, from }: CollectorOptions): Promise { + const dsl = { + index: [APM_INDICES, LOGS_INDICES, METRICS_INDICES], + size: 1000, + collapse: { field: 'host.hostname' }, + sort: [{ _score: 'desc' }, { '@timestamp': 'desc' }], + _source: false, + fields: [ + '@timestamp', + 'cloud.*', + 'container.*', + 'host.hostname', + 'kubernetes.*', + 'orchestrator.cluster.name', + ], + query: { + bool: { + filter: [ + { + range: { + '@timestamp': { + gte: from, + }, + }, + }, + ], + must: [{ exists: { field: 'host.hostname' } }], + should: [ + { exists: { field: 'kubernetes.node.name' } }, + { exists: { field: 'kubernetes.pod.uid' } }, + { exists: { field: 'container.id' } }, + ], + }, + }, + }; + + const esResponse = await client.search(dsl); + + const hosts = esResponse.hits.hits.reduce((acc: Asset[], hit: any) => { + const { fields = {} } = hit; + const hostName = fields['host.hostname']; + const k8sNode = fields['kubernetes.node.name']; + const k8sPod = fields['kubernetes.pod.uid']; + + const hostEan = `host:${k8sNode || hostName}`; + + const host: Asset = { + '@timestamp': new Date().toISOString(), + 'asset.kind': 'host', + 'asset.id': k8sNode || hostName, + 'asset.name': k8sNode || hostName, + 'asset.ean': hostEan, + }; + + if (fields['cloud.provider']) { + host['cloud.provider'] = fields['cloud.provider']; + } + + if (fields['cloud.instance.id']) { + host['cloud.instance.id'] = fields['cloud.instance.id']; + } + + if (fields['cloud.service.name']) { + host['cloud.service.name'] = fields['cloud.service.name']; + } + + if (fields['cloud.region']) { + host['cloud.region'] = fields['cloud.region']; + } + + if (fields['orchestrator.cluster.name']) { + host['orchestrator.cluster.name'] = fields['orchestrator.cluster.name']; + } + + if (k8sPod) { + host['asset.children'] = [`pod:${k8sPod}`]; + } + + acc.push(host); + + return acc; + }, []); + + return hosts; +} diff --git a/x-pack/plugins/asset_manager/server/lib/implicit_collection/collectors/index.ts b/x-pack/plugins/asset_manager/server/lib/implicit_collection/collectors/index.ts new file mode 100644 index 0000000000000..79b53597a9a40 --- /dev/null +++ b/x-pack/plugins/asset_manager/server/lib/implicit_collection/collectors/index.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { ElasticsearchClient } from '@kbn/core/server'; +import { Asset } from '../../../../common/types_api'; + +export interface CollectorOptions { + client: ElasticsearchClient; + from: number; +} + +export type Collector = (opts: CollectorOptions) => Promise; + +export { collectContainers } from './containers'; +export { collectHosts } from './hosts'; +export { collectPods } from './pods'; +export { collectServices } from './services'; diff --git a/x-pack/plugins/asset_manager/server/lib/implicit_collection/collectors/pods.ts b/x-pack/plugins/asset_manager/server/lib/implicit_collection/collectors/pods.ts new file mode 100644 index 0000000000000..4bb983acfca6e --- /dev/null +++ b/x-pack/plugins/asset_manager/server/lib/implicit_collection/collectors/pods.ts @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { APM_INDICES, LOGS_INDICES, METRICS_INDICES } from '../../../constants'; +import { Asset } from '../../../../common/types_api'; +import { CollectorOptions } from '.'; + +export async function collectPods({ client, from }: CollectorOptions) { + const dsl = { + index: [APM_INDICES, LOGS_INDICES, METRICS_INDICES], + size: 1000, + collapse: { + field: 'kubernetes.pod.uid', + }, + sort: [{ '@timestamp': 'desc' }], + _source: false, + fields: [ + 'kubernetes.*', + 'cloud.provider', + 'orchestrator.cluster.name', + 'host.name', + 'host.hostname', + ], + query: { + bool: { + filter: [ + { + range: { + '@timestamp': { + gte: from, + }, + }, + }, + ], + must: [ + { exists: { field: 'kubernetes.pod.uid' } }, + { exists: { field: 'kubernetes.node.name' } }, + ], + }, + }, + }; + + const esResponse = await client.search(dsl); + + const pods = esResponse.hits.hits.reduce((acc: Asset[], hit: any) => { + const { fields = {} } = hit; + const podUid = fields['kubernetes.pod.uid']; + const nodeName = fields['kubernetes.node.name']; + const clusterName = fields['orchestrator.cluster.name']; + + const pod: Asset = { + '@timestamp': new Date().toISOString(), + 'asset.kind': 'pod', + 'asset.id': podUid, + 'asset.ean': `pod:${podUid}`, + 'asset.parents': [`host:${nodeName}`], + }; + + if (fields['cloud.provider']) { + pod['cloud.provider'] = fields['cloud.provider']; + } + + if (clusterName) { + pod['orchestrator.cluster.name'] = clusterName; + } + + acc.push(pod); + + return acc; + }, []); + + return pods; +} diff --git a/x-pack/plugins/asset_manager/server/lib/implicit_collection/collectors/services.ts b/x-pack/plugins/asset_manager/server/lib/implicit_collection/collectors/services.ts new file mode 100644 index 0000000000000..e1fd0c7a1a042 --- /dev/null +++ b/x-pack/plugins/asset_manager/server/lib/implicit_collection/collectors/services.ts @@ -0,0 +1,110 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { APM_INDICES } from '../../../constants'; +import { Asset } from '../../../../common/types_api'; +import { CollectorOptions } from '.'; + +const MISSING_KEY = '__unknown__'; + +export async function collectServices({ client, from }: CollectorOptions): Promise { + const dsl = { + index: APM_INDICES, + size: 0, + sort: [ + { + '@timestamp': 'desc', + }, + ], + _source: false, + query: { + bool: { + filter: [ + { + range: { + '@timestamp': { + gte: from, + }, + }, + }, + ], + must: [ + { + exists: { + field: 'service.name', + }, + }, + ], + }, + }, + aggs: { + service_environment: { + multi_terms: { + size: 100, + terms: [ + { + field: 'service.name', + }, + { + field: 'service.environment', + missing: MISSING_KEY, + }, + ], + }, + aggs: { + container_host: { + multi_terms: { + size: 100, + terms: [ + { field: 'container.id', missing: MISSING_KEY }, + { field: 'host.hostname', missing: MISSING_KEY }, + ], + }, + }, + }, + }, + }, + }; + + const esResponse = await client.search(dsl); + const serviceEnvironment = esResponse.aggregations?.service_environment as { buckets: any[] }; + + const services = (serviceEnvironment?.buckets ?? []).reduce((acc: Asset[], hit: any) => { + const [serviceName, environment] = hit.key; + const containerHosts = hit.container_host.buckets; + + const service: Asset = { + '@timestamp': new Date().toISOString(), + 'asset.kind': 'service', + 'asset.id': serviceName, + 'asset.ean': `service:${serviceName}`, + 'asset.references': [], + 'asset.parents': [], + }; + + if (environment !== MISSING_KEY) { + service['service.environment'] = environment; + } + + containerHosts.forEach((nestedHit: any) => { + const [containerId, hostname] = nestedHit.key; + if (containerId !== MISSING_KEY) { + (service['asset.parents'] as string[]).push(`container:${containerId}`); + } + + if (hostname !== MISSING_KEY) { + (service['asset.references'] as string[]).push(`host:${hostname}`); + } + }); + + acc.push(service); + + return acc; + }, []); + + return services; +} diff --git a/x-pack/plugins/asset_manager/server/lib/implicit_collection/index.ts b/x-pack/plugins/asset_manager/server/lib/implicit_collection/index.ts new file mode 100644 index 0000000000000..c33d5f4877d80 --- /dev/null +++ b/x-pack/plugins/asset_manager/server/lib/implicit_collection/index.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { ElasticsearchClient, Logger } from '@kbn/core/server'; + +import { collectContainers, collectHosts, collectPods, collectServices } from './collectors'; +import { CollectorRunner } from './collector_runner'; + +export interface ImplicitCollectionOptions { + inputClient: ElasticsearchClient; + outputClient: ElasticsearchClient; + intervalMs: number; + logger: Logger; +} + +export function startImplicitCollection(options: ImplicitCollectionOptions): () => void { + const runner = new CollectorRunner(options); + runner.registerCollector('containers', collectContainers); + runner.registerCollector('hosts', collectHosts); + runner.registerCollector('pods', collectPods); + runner.registerCollector('services', collectServices); + + const timer = setInterval(async () => { + options.logger.info('Starting execution'); + try { + await runner.run(); + options.logger.info('Execution ended successfully'); + } catch (err) { + options.logger.info(`Execution ended with error: ${err}`); + } + }, options.intervalMs); + + return () => { + options.logger.debug('Stopping periodic collection'); + clearInterval(timer); + }; +} diff --git a/x-pack/plugins/asset_manager/server/lib/sample_assets.ts b/x-pack/plugins/asset_manager/server/lib/sample_assets.ts index f49300e4663e6..762438b9d1b4b 100644 --- a/x-pack/plugins/asset_manager/server/lib/sample_assets.ts +++ b/x-pack/plugins/asset_manager/server/lib/sample_assets.ts @@ -30,6 +30,7 @@ export function getSampleAssetDocs({ const sampleK8sClusters: AssetWithoutTimestamp[] = [ { 'asset.type': 'k8s.cluster', + 'asset.kind': 'cluster', 'asset.id': 'cluster-001', 'asset.name': 'Cluster 001 (AWS EKS)', 'asset.ean': 'k8s.cluster:cluster-001', @@ -42,6 +43,7 @@ const sampleK8sClusters: AssetWithoutTimestamp[] = [ }, { 'asset.type': 'k8s.cluster', + 'asset.kind': 'cluster', 'asset.id': 'cluster-002', 'asset.name': 'Cluster 002 (Azure AKS)', 'asset.ean': 'k8s.cluster:cluster-002', @@ -57,6 +59,7 @@ const sampleK8sClusters: AssetWithoutTimestamp[] = [ const sampleK8sNodes: AssetWithoutTimestamp[] = [ { 'asset.type': 'k8s.node', + 'asset.kind': 'host', 'asset.id': 'node-101', 'asset.name': 'k8s-node-101-aws', 'asset.ean': 'k8s.node:node-101', @@ -70,6 +73,7 @@ const sampleK8sNodes: AssetWithoutTimestamp[] = [ }, { 'asset.type': 'k8s.node', + 'asset.kind': 'host', 'asset.id': 'node-102', 'asset.name': 'k8s-node-102-aws', 'asset.ean': 'k8s.node:node-102', @@ -83,6 +87,7 @@ const sampleK8sNodes: AssetWithoutTimestamp[] = [ }, { 'asset.type': 'k8s.node', + 'asset.kind': 'host', 'asset.id': 'node-103', 'asset.name': 'k8s-node-103-aws', 'asset.ean': 'k8s.node:node-103', @@ -99,6 +104,7 @@ const sampleK8sNodes: AssetWithoutTimestamp[] = [ const sampleK8sPods: AssetWithoutTimestamp[] = [ { 'asset.type': 'k8s.pod', + 'asset.kind': 'pod', 'asset.id': 'pod-200xrg1', 'asset.name': 'k8s-pod-200xrg1-aws', 'asset.ean': 'k8s.pod:pod-200xrg1', @@ -107,6 +113,7 @@ const sampleK8sPods: AssetWithoutTimestamp[] = [ }, { 'asset.type': 'k8s.pod', + 'asset.kind': 'pod', 'asset.id': 'pod-200dfp2', 'asset.name': 'k8s-pod-200dfp2-aws', 'asset.ean': 'k8s.pod:pod-200dfp2', @@ -114,6 +121,7 @@ const sampleK8sPods: AssetWithoutTimestamp[] = [ }, { 'asset.type': 'k8s.pod', + 'asset.kind': 'pod', 'asset.id': 'pod-200wwc3', 'asset.name': 'k8s-pod-200wwc3-aws', 'asset.ean': 'k8s.pod:pod-200wwc3', @@ -121,6 +129,7 @@ const sampleK8sPods: AssetWithoutTimestamp[] = [ }, { 'asset.type': 'k8s.pod', + 'asset.kind': 'pod', 'asset.id': 'pod-200naq4', 'asset.name': 'k8s-pod-200naq4-aws', 'asset.ean': 'k8s.pod:pod-200naq4', @@ -128,6 +137,7 @@ const sampleK8sPods: AssetWithoutTimestamp[] = [ }, { 'asset.type': 'k8s.pod', + 'asset.kind': 'pod', 'asset.id': 'pod-200ohr5', 'asset.name': 'k8s-pod-200ohr5-aws', 'asset.ean': 'k8s.pod:pod-200ohr5', @@ -135,6 +145,7 @@ const sampleK8sPods: AssetWithoutTimestamp[] = [ }, { 'asset.type': 'k8s.pod', + 'asset.kind': 'pod', 'asset.id': 'pod-200yyx6', 'asset.name': 'k8s-pod-200yyx6-aws', 'asset.ean': 'k8s.pod:pod-200yyx6', @@ -142,6 +153,7 @@ const sampleK8sPods: AssetWithoutTimestamp[] = [ }, { 'asset.type': 'k8s.pod', + 'asset.kind': 'pod', 'asset.id': 'pod-200psd7', 'asset.name': 'k8s-pod-200psd7-aws', 'asset.ean': 'k8s.pod:pod-200psd7', @@ -149,6 +161,7 @@ const sampleK8sPods: AssetWithoutTimestamp[] = [ }, { 'asset.type': 'k8s.pod', + 'asset.kind': 'pod', 'asset.id': 'pod-200wmc8', 'asset.name': 'k8s-pod-200wmc8-aws', 'asset.ean': 'k8s.pod:pod-200wmc8', @@ -156,6 +169,7 @@ const sampleK8sPods: AssetWithoutTimestamp[] = [ }, { 'asset.type': 'k8s.pod', + 'asset.kind': 'pod', 'asset.id': 'pod-200ugg9', 'asset.name': 'k8s-pod-200ugg9-aws', 'asset.ean': 'k8s.pod:pod-200ugg9', @@ -166,6 +180,7 @@ const sampleK8sPods: AssetWithoutTimestamp[] = [ const sampleCircularReferences: AssetWithoutTimestamp[] = [ { 'asset.type': 'k8s.node', + 'asset.kind': 'host', 'asset.id': 'node-203', 'asset.name': 'k8s-node-203-aws', 'asset.ean': 'k8s.node:node-203', @@ -179,6 +194,7 @@ const sampleCircularReferences: AssetWithoutTimestamp[] = [ }, { 'asset.type': 'k8s.pod', + 'asset.kind': 'pod', 'asset.id': 'pod-203ugg5', 'asset.name': 'k8s-pod-203ugg5-aws', 'asset.ean': 'k8s.pod:pod-203ugg5', @@ -186,6 +202,7 @@ const sampleCircularReferences: AssetWithoutTimestamp[] = [ }, { 'asset.type': 'k8s.pod', + 'asset.kind': 'pod', 'asset.id': 'pod-203ugg9', 'asset.name': 'k8s-pod-203ugg9-aws', 'asset.ean': 'k8s.pod:pod-203ugg9', diff --git a/x-pack/plugins/asset_manager/server/plugin.ts b/x-pack/plugins/asset_manager/server/plugin.ts index d81d3f1f87b63..07b41bc5780e5 100644 --- a/x-pack/plugins/asset_manager/server/plugin.ts +++ b/x-pack/plugins/asset_manager/server/plugin.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { schema } from '@kbn/config-schema'; +import { schema, TypeOf } from '@kbn/config-schema'; import { Plugin, CoreSetup, @@ -14,27 +14,54 @@ import { PluginInitializerContext, PluginConfigDescriptor, Logger, + ElasticsearchClient, } from '@kbn/core/server'; + import { upsertTemplate } from './lib/manage_index_templates'; +import { startImplicitCollection } from './lib/implicit_collection'; import { setupRoutes } from './routes'; import { assetsIndexTemplateConfig } from './templates/assets_template'; export type AssetManagerServerPluginSetup = ReturnType; -export interface AssetManagerConfig { - alphaEnabled?: boolean; -} + +const configSchema = schema.object({ + alphaEnabled: schema.maybe(schema.boolean()), + implicitCollection: schema.maybe( + schema.object({ + enabled: schema.boolean({ defaultValue: true }), + interval: schema.duration({ defaultValue: '5m' }), + input: schema.maybe( + schema.object({ + hosts: schema.string(), + username: schema.string(), + password: schema.string(), + }) + ), + output: schema.maybe( + schema.object({ + hosts: schema.string(), + username: schema.string(), + password: schema.string(), + }) + ), + }) + ), +}); + +export type AssetManagerConfig = TypeOf; export const config: PluginConfigDescriptor = { - schema: schema.object({ - alphaEnabled: schema.maybe(schema.boolean()), - }), + schema: configSchema, }; export class AssetManagerServerPlugin implements Plugin { + private context: PluginInitializerContext; + private stopImplicitCollection?: () => void; public config: AssetManagerConfig; public logger: Logger; constructor(context: PluginInitializerContext) { + this.context = context; this.config = context.config.get(); this.logger = context.logger.get(); } @@ -66,7 +93,47 @@ export class AssetManagerServerPlugin implements Plugin = new Set(); for (let i = 0; i < sampleAssetDocs.length; i++) { - sampleTypeSet.add(sampleAssetDocs[i]['asset.type']); + sampleTypeSet.add(sampleAssetDocs[i]['asset.type']!); } const sampleTypes = Array.from(sampleTypeSet); if (sampleTypes.length <= 2) { @@ -107,7 +107,7 @@ export default function ({ getService }: FtrProviderContext) { // Track a reference to how many docs should be returned for these two types const samplesForFilteredTypes = sampleAssetDocs.filter((doc) => - filterByTypes.includes(doc['asset.type']) + filterByTypes.includes(doc['asset.type']!) ); expect(samplesForFilteredTypes.length).to.be.lessThan(sampleAssetDocs.length);