Skip to content

Commit

Permalink
[Asset Manager] run implicit collection periodically (elastic#156830)
Browse files Browse the repository at this point in the history
Closes elastic#156757

Introduce background implicit collection in the asset_manager plugin.
The process can be configured with the following kibana settings:

```
xpack.assetManager:
  implicitCollection:
    enabled: true
    interval: 30s

    # elasticsearch cluster we should extract signals from
    input:
      hosts: http://input:9200
      username: ...
      password: ...

    # elasticsearch cluster we should write assets to
    output:
      hosts: http://output:9200
      username: ...
      password: ...
```

---------

Co-authored-by: kibanamachine <[email protected]>
  • Loading branch information
klacabane and kibanamachine committed Jul 19, 2023
1 parent e98abd0 commit 0b23894
Show file tree
Hide file tree
Showing 17 changed files with 721 additions and 14 deletions.
9 changes: 6 additions & 3 deletions x-pack/plugins/asset_manager/common/types_api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ export const assetTypeRT = rt.union([

export type AssetType = rt.TypeOf<typeof assetTypeRT>;

export type AssetKind = 'unknown' | 'node';
export type AssetKind = 'cluster' | 'host' | 'pod' | 'container' | 'service';
export type AssetStatus =
| 'CREATING'
| 'ACTIVE'
Expand Down Expand Up @@ -47,17 +47,20 @@ export interface ECSDocument extends WithTimestamp {
'orchestrator.cluster.version'?: string;

'cloud.provider'?: CloudProviderName;
'cloud.instance.id'?: string;
'cloud.region'?: string;
'cloud.service.name'?: string;

'service.environment'?: string;
}

export interface Asset extends ECSDocument {
'asset.collection_version'?: string;
'asset.ean': string;
'asset.id': string;
'asset.kind'?: AssetKind;
'asset.kind': AssetKind;
'asset.name'?: string;
'asset.type': AssetType;
'asset.type'?: AssetType;
'asset.status'?: AssetStatus;
'asset.parents'?: string | string[];
'asset.children'?: string | string[];
Expand Down
4 changes: 4 additions & 0 deletions x-pack/plugins/asset_manager/server/constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,7 @@

export const ASSETS_INDEX_PREFIX = 'assets';
export const ASSET_MANAGER_API_BASE = '/api/asset-manager';

export const LOGS_INDICES = 'logs-*,filebeat-*';
export const APM_INDICES = 'traces-*,apm*,metrics-apm*,logs-apm*';
export const METRICS_INDICES = 'metrics-*,metricbeat-*';
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ describe('getAllRelatedAssets', () => {
const primaryAsset: AssetWithoutTimestamp = {
'asset.ean': 'primary-which-does-not-exist',
'asset.type': 'k8s.pod',
'asset.kind': 'pod',
'asset.id': uuid(),
};

Expand All @@ -53,6 +54,7 @@ describe('getAllRelatedAssets', () => {
const primaryAssetWithoutParents: AssetWithoutTimestamp = {
'asset.ean': 'primary-ean',
'asset.type': 'k8s.pod',
'asset.kind': 'pod',
'asset.id': uuid(),
'asset.parents': [],
};
Expand Down Expand Up @@ -82,12 +84,14 @@ describe('getAllRelatedAssets', () => {
const parentAsset: AssetWithoutTimestamp = {
'asset.ean': 'primary-ean',
'asset.type': 'k8s.pod',
'asset.kind': 'pod',
'asset.id': uuid(),
};

const primaryAssetWithDirectParent: AssetWithoutTimestamp = {
'asset.ean': 'primary-ean',
'asset.type': 'k8s.pod',
'asset.kind': 'pod',
'asset.id': uuid(),
'asset.parents': [parentAsset['asset.ean']],
};
Expand Down Expand Up @@ -124,13 +128,15 @@ describe('getAllRelatedAssets', () => {
const primaryAssetWithIndirectParent: AssetWithoutTimestamp = {
'asset.ean': 'primary-ean',
'asset.type': 'k8s.pod',
'asset.kind': 'pod',
'asset.id': uuid(),
'asset.parents': [],
};

const parentAsset: AssetWithoutTimestamp = {
'asset.ean': 'primary-ean',
'asset.type': 'k8s.pod',
'asset.kind': 'pod',
'asset.id': uuid(),
'asset.children': [primaryAssetWithIndirectParent['asset.ean']],
};
Expand Down Expand Up @@ -167,20 +173,23 @@ describe('getAllRelatedAssets', () => {
const directlyReferencedParent: AssetWithoutTimestamp = {
'asset.ean': 'directly-referenced-parent-ean',
'asset.type': 'k8s.pod',
'asset.kind': 'pod',
'asset.id': uuid(),
'asset.children': [],
};

const primaryAsset: AssetWithoutTimestamp = {
'asset.ean': 'primary-ean',
'asset.type': 'k8s.pod',
'asset.kind': 'pod',
'asset.id': uuid(),
'asset.parents': [directlyReferencedParent['asset.ean']],
};

const indirectlyReferencedParent: AssetWithoutTimestamp = {
'asset.ean': 'indirectly-referenced-parent-ean',
'asset.type': 'k8s.pod',
'asset.kind': 'pod',
'asset.id': uuid(),
'asset.children': [primaryAsset['asset.ean']],
};
Expand Down Expand Up @@ -221,12 +230,14 @@ describe('getAllRelatedAssets', () => {
const parentAsset: AssetWithoutTimestamp = {
'asset.ean': 'parent-ean',
'asset.type': 'k8s.pod',
'asset.kind': 'pod',
'asset.id': uuid(),
};

const primaryAsset: AssetWithoutTimestamp = {
'asset.ean': 'primary-ean',
'asset.type': 'k8s.pod',
'asset.kind': 'pod',
'asset.id': uuid(),
};

Expand Down Expand Up @@ -266,47 +277,54 @@ describe('getAllRelatedAssets', () => {
const distance6Parent: AssetWithoutTimestamp = {
'asset.ean': 'parent-5-ean',
'asset.type': 'k8s.pod',
'asset.kind': 'pod',
'asset.id': uuid(),
};

const distance5Parent: AssetWithoutTimestamp = {
'asset.ean': 'parent-5-ean',
'asset.type': 'k8s.pod',
'asset.kind': 'pod',
'asset.id': uuid(),
'asset.parents': [distance6Parent['asset.ean']],
};

const distance4Parent: AssetWithoutTimestamp = {
'asset.ean': 'parent-4-ean',
'asset.type': 'k8s.pod',
'asset.kind': 'pod',
'asset.id': uuid(),
'asset.parents': [distance5Parent['asset.ean']],
};

const distance3Parent: AssetWithoutTimestamp = {
'asset.ean': 'parent-3-ean',
'asset.type': 'k8s.pod',
'asset.kind': 'pod',
'asset.id': uuid(),
'asset.parents': [distance4Parent['asset.ean']],
};

const distance2Parent: AssetWithoutTimestamp = {
'asset.ean': 'parent-2-ean',
'asset.type': 'k8s.pod',
'asset.kind': 'pod',
'asset.id': uuid(),
'asset.parents': [distance3Parent['asset.ean']],
};

const distance1Parent: AssetWithoutTimestamp = {
'asset.ean': 'parent-1-ean',
'asset.type': 'k8s.pod',
'asset.kind': 'pod',
'asset.id': uuid(),
'asset.parents': [distance2Parent['asset.ean']],
};

const primaryAsset: AssetWithoutTimestamp = {
'asset.ean': 'primary-ean',
'asset.type': 'k8s.pod',
'asset.kind': 'pod',
'asset.id': uuid(),
'asset.parents': [distance1Parent['asset.ean']],
};
Expand Down Expand Up @@ -368,26 +386,30 @@ describe('getAllRelatedAssets', () => {
const distance3Parent: AssetWithoutTimestamp = {
'asset.ean': 'parent-3-ean',
'asset.type': 'k8s.pod',
'asset.kind': 'pod',
'asset.id': uuid(),
};

const distance2Parent: AssetWithoutTimestamp = {
'asset.ean': 'parent-2-ean',
'asset.type': 'k8s.pod',
'asset.kind': 'pod',
'asset.id': uuid(),
'asset.parents': [distance3Parent['asset.ean']],
};

const distance1Parent: AssetWithoutTimestamp = {
'asset.ean': 'parent-1-ean',
'asset.type': 'k8s.pod',
'asset.kind': 'pod',
'asset.id': uuid(),
'asset.parents': [distance2Parent['asset.ean']],
};

const primaryAsset: AssetWithoutTimestamp = {
'asset.ean': 'primary-ean',
'asset.type': 'k8s.pod',
'asset.kind': 'pod',
'asset.id': uuid(),
'asset.parents': [distance1Parent['asset.ean']],
};
Expand Down Expand Up @@ -435,44 +457,51 @@ describe('getAllRelatedAssets', () => {
const distance2ParentA: AssetWithoutTimestamp = {
'asset.ean': 'parent-2-ean-a',
'asset.type': 'k8s.pod',
'asset.kind': 'pod',
'asset.id': uuid(),
};

const distance2ParentB: AssetWithoutTimestamp = {
'asset.ean': 'parent-2-ean-b',
'asset.type': 'k8s.pod',
'asset.kind': 'pod',
'asset.id': uuid(),
};

const distance2ParentC: AssetWithoutTimestamp = {
'asset.ean': 'parent-2-ean-c',
'asset.type': 'k8s.pod',
'asset.kind': 'pod',
'asset.id': uuid(),
};

const distance2ParentD: AssetWithoutTimestamp = {
'asset.ean': 'parent-2-ean-d',
'asset.type': 'k8s.pod',
'asset.kind': 'pod',
'asset.id': uuid(),
};

const distance1ParentA: AssetWithoutTimestamp = {
'asset.ean': 'parent-1-ean-a',
'asset.type': 'k8s.pod',
'asset.kind': 'pod',
'asset.id': uuid(),
'asset.parents': [distance2ParentA['asset.ean'], distance2ParentB['asset.ean']],
};

const distance1ParentB: AssetWithoutTimestamp = {
'asset.ean': 'parent-1-ean-b',
'asset.type': 'k8s.pod',
'asset.kind': 'pod',
'asset.id': uuid(),
'asset.parents': [distance2ParentC['asset.ean'], distance2ParentD['asset.ean']],
};

const primaryAsset: AssetWithoutTimestamp = {
'asset.ean': 'primary-ean',
'asset.type': 'k8s.pod',
'asset.kind': 'pod',
'asset.id': uuid(),
'asset.parents': [distance1ParentA['asset.ean'], distance1ParentB['asset.ean']],
};
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ export async function getAllRelatedAssets(
return {
primary,
[relation]: type.length
? relatedAssets.filter((asset) => type.includes(asset['asset.type']))
? relatedAssets.filter((asset) => asset['asset.type'] && type.includes(asset['asset.type']))
: relatedAssets,
};
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import expect from '@kbn/expect';
import { ElasticsearchClient, Logger } from '@kbn/core/server';
import { CollectorRunner } from './collector_runner';
import { CollectorOptions } from './collectors';
import { Asset } from '../../../common/types_api';

const getMockClient = () => ({
bulk: jest.fn().mockResolvedValue({ errors: false }),
});

const getMockLogger = () =>
({
info: jest.fn(),
error: jest.fn(),
} as unknown as Logger);

describe(__filename, () => {
it('runs registered collectors', async () => {
const runner = new CollectorRunner({
inputClient: getMockClient() as unknown as ElasticsearchClient,
outputClient: getMockClient() as unknown as ElasticsearchClient,
logger: getMockLogger(),
intervalMs: 1,
});

const collector1 = jest.fn(async (opts: CollectorOptions) => {
return [];
});
const collector2 = jest.fn(async (opts: CollectorOptions) => {
return [];
});

runner.registerCollector('foo', collector1);
runner.registerCollector('foo', collector2);

await runner.run();

expect(collector1.mock.calls).to.have.length(1);
expect(collector2.mock.calls).to.have.length(1);
});

it('is resilient to failing collectors', async () => {
const runner = new CollectorRunner({
outputClient: getMockClient() as unknown as ElasticsearchClient,
inputClient: getMockClient() as unknown as ElasticsearchClient,
logger: getMockLogger(),
intervalMs: 1,
});

const collector1 = jest.fn(async (opts: CollectorOptions) => {
throw new Error('no');
});
const collector2 = jest.fn(async (opts: CollectorOptions) => {
return [];
});

runner.registerCollector('foo', collector1);
runner.registerCollector('foo', collector2);

await runner.run();

expect(collector1.mock.calls).to.have.length(1);
expect(collector2.mock.calls).to.have.length(1);
});

it('stores collectors results in elasticsearch', async () => {
const outputClient = getMockClient();
const runner = new CollectorRunner({
outputClient: outputClient as unknown as ElasticsearchClient,
inputClient: getMockClient() as unknown as ElasticsearchClient,
logger: getMockLogger(),
intervalMs: 1,
});

const collector = jest.fn(async (opts: CollectorOptions) => {
return [
{ 'asset.kind': 'container', 'asset.ean': 'foo' },
{ 'asset.kind': 'pod', 'asset.ean': 'bar' },
] as Asset[];
});

runner.registerCollector('foo', collector);

await runner.run();

expect(outputClient.bulk.mock.calls[0][0]).to.eql({
body: [
{ create: { _index: 'assets-container-default' } },
{ 'asset.kind': 'container', 'asset.ean': 'foo' },
{ create: { _index: 'assets-pod-default' } },
{ 'asset.kind': 'pod', 'asset.ean': 'bar' },
],
});
});
});
Loading

0 comments on commit 0b23894

Please sign in to comment.