Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[asset manager] remove sync apm span #162180

Merged
merged 2 commits into from
Jul 19, 2023
Merged

[asset manager] remove sync apm span #162180

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
20d9fc5
remove sync apm spans
klacabane Jul 18, 2023
edd8012
Merge branch 'feat/obs-asset-manager-demo' into 161887-remove-sync-ap…
klacabane Jul 18, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 22 additions & 28 deletions x-pack/plugins/asset_manager/server/lib/collectors/containers.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,11 @@
import { estypes } from '@elastic/elasticsearch';
import { Asset } from '../../../common/types_api';
import { CollectorOptions, QUERY_MAX_SIZE } from '.';
import { withSpan } from './helpers';

export async function collectContainers({
client,
from,
to,
transaction,
sourceIndices,
afterKey,
}: CollectorOptions) {
Expand Down Expand Up @@ -62,36 +60,32 @@ export async function collectContainers({

const esResponse = await client.search(dsl);

const result = withSpan({ transaction, name: 'processing_response' }, async () => {
const assets = esResponse.hits.hits.reduce<Asset[]>((acc: Asset[], hit: any) => {
const { fields = {} } = hit;
const containerId = fields['container.id'];
const podUid = fields['kubernetes.pod.uid'];
const nodeName = fields['kubernetes.node.name'];
const assets = esResponse.hits.hits.reduce<Asset[]>((acc: Asset[], hit: any) => {
const { fields = {} } = hit;
const containerId = fields['container.id'];
const podUid = fields['kubernetes.pod.uid'];
const nodeName = fields['kubernetes.node.name'];

const parentEan = podUid ? `pod:${podUid}` : `host:${fields['host.hostname']}`;
const parentEan = podUid ? `pod:${podUid}` : `host:${fields['host.hostname']}`;

const container: Asset = {
'@timestamp': new Date().toISOString(),
'asset.kind': 'container',
'asset.id': containerId,
'asset.ean': `container:${containerId}`,
'asset.parents': [parentEan],
};
const container: Asset = {
'@timestamp': new Date().toISOString(),
'asset.kind': 'container',
'asset.id': containerId,
'asset.ean': `container:${containerId}`,
'asset.parents': [parentEan],
};

if (nodeName) {
container['asset.references'] = [`host:${nodeName}`];
}
if (nodeName) {
container['asset.references'] = [`host:${nodeName}`];
}

acc.push(container);
acc.push(container);

return acc;
}, []);
return acc;
}, []);

const hitsLen = esResponse.hits.hits.length;
const next = hitsLen === QUERY_MAX_SIZE ? esResponse.hits.hits[hitsLen - 1].sort : undefined;
return { assets, afterKey: next };
});

return result;
const hitsLen = esResponse.hits.hits.length;
const next = hitsLen === QUERY_MAX_SIZE ? esResponse.hits.hits[hitsLen - 1].sort : undefined;
return { assets, afterKey: next };
}
80 changes: 37 additions & 43 deletions x-pack/plugins/asset_manager/server/lib/collectors/hosts.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,11 @@
import { estypes } from '@elastic/elasticsearch';
import { Asset } from '../../../common/types_api';
import { CollectorOptions, QUERY_MAX_SIZE } from '.';
import { withSpan } from './helpers';

export async function collectHosts({
client,
from,
to,
transaction,
sourceIndices,
afterKey,
}: CollectorOptions) {
Expand Down Expand Up @@ -61,56 +59,52 @@ export async function collectHosts({

const esResponse = await client.search(dsl);

const result = withSpan({ transaction, name: 'processing_response' }, async () => {
const assets = esResponse.hits.hits.reduce<Asset[]>((acc: Asset[], hit: any) => {
const { fields = {} } = hit;
const hostName = fields['host.hostname'];
const k8sNode = fields['kubernetes.node.name'];
const k8sPod = fields['kubernetes.pod.uid'];
const assets = esResponse.hits.hits.reduce<Asset[]>((acc: Asset[], hit: any) => {
const { fields = {} } = hit;
const hostName = fields['host.hostname'];
const k8sNode = fields['kubernetes.node.name'];
const k8sPod = fields['kubernetes.pod.uid'];

const hostEan = `host:${k8sNode || hostName}`;
const hostEan = `host:${k8sNode || hostName}`;

const host: Asset = {
'@timestamp': new Date().toISOString(),
'asset.kind': 'host',
'asset.id': k8sNode || hostName,
'asset.name': k8sNode || hostName,
'asset.ean': hostEan,
};
const host: Asset = {
'@timestamp': new Date().toISOString(),
'asset.kind': 'host',
'asset.id': k8sNode || hostName,
'asset.name': k8sNode || hostName,
'asset.ean': hostEan,
};

if (fields['cloud.provider']) {
host['cloud.provider'] = fields['cloud.provider'];
}
if (fields['cloud.provider']) {
host['cloud.provider'] = fields['cloud.provider'];
}

if (fields['cloud.instance.id']) {
host['cloud.instance.id'] = fields['cloud.instance.id'];
}
if (fields['cloud.instance.id']) {
host['cloud.instance.id'] = fields['cloud.instance.id'];
}

if (fields['cloud.service.name']) {
host['cloud.service.name'] = fields['cloud.service.name'];
}
if (fields['cloud.service.name']) {
host['cloud.service.name'] = fields['cloud.service.name'];
}

if (fields['cloud.region']) {
host['cloud.region'] = fields['cloud.region'];
}
if (fields['cloud.region']) {
host['cloud.region'] = fields['cloud.region'];
}

if (fields['orchestrator.cluster.name']) {
host['orchestrator.cluster.name'] = fields['orchestrator.cluster.name'];
}
if (fields['orchestrator.cluster.name']) {
host['orchestrator.cluster.name'] = fields['orchestrator.cluster.name'];
}

if (k8sPod) {
host['asset.children'] = [`pod:${k8sPod}`];
}
if (k8sPod) {
host['asset.children'] = [`pod:${k8sPod}`];
}

acc.push(host);
acc.push(host);

return acc;
}, []);
return acc;
}, []);

const hitsLen = esResponse.hits.hits.length;
const next = hitsLen === QUERY_MAX_SIZE ? esResponse.hits.hits[hitsLen - 1].sort : undefined;
return { assets, afterKey: next };
});

return result;
const hitsLen = esResponse.hits.hits.length;
const next = hitsLen === QUERY_MAX_SIZE ? esResponse.hits.hits[hitsLen - 1].sort : undefined;
return { assets, afterKey: next };
}
2 changes: 0 additions & 2 deletions x-pack/plugins/asset_manager/server/lib/collectors/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
* 2.0.
*/

import { Transaction } from 'elastic-apm-node';
import { estypes } from '@elastic/elasticsearch';
import { ElasticsearchClient } from '@kbn/core/server';
import { AssetManagerConfig } from '../../types';
Expand All @@ -19,7 +18,6 @@ export interface CollectorOptions {
client: ElasticsearchClient;
from: number;
to: number;
transaction?: Transaction | null;
sourceIndices: AssetManagerConfig['sourceIndices'];
afterKey?: estypes.SortResults;
}
Expand Down
62 changes: 25 additions & 37 deletions x-pack/plugins/asset_manager/server/lib/collectors/pods.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,16 +8,8 @@
import { estypes } from '@elastic/elasticsearch';
import { Asset } from '../../../common/types_api';
import { CollectorOptions, QUERY_MAX_SIZE } from '.';
import { withSpan } from './helpers';

export async function collectPods({
client,
from,
to,
transaction,
sourceIndices,
afterKey,
}: CollectorOptions) {
export async function collectPods({ client, from, to, sourceIndices, afterKey }: CollectorOptions) {
const { metrics, logs, traces } = sourceIndices;
const dsl: estypes.SearchRequest = {
index: [metrics, logs, traces],
Expand Down Expand Up @@ -60,38 +52,34 @@ export async function collectPods({

const esResponse = await client.search(dsl);

const result = withSpan({ transaction, name: 'processing_response' }, async () => {
const assets = esResponse.hits.hits.reduce<Asset[]>((acc: Asset[], hit: any) => {
const { fields = {} } = hit;
const podUid = fields['kubernetes.pod.uid'];
const nodeName = fields['kubernetes.node.name'];
const clusterName = fields['orchestrator.cluster.name'];
const assets = esResponse.hits.hits.reduce<Asset[]>((acc: Asset[], hit: any) => {
const { fields = {} } = hit;
const podUid = fields['kubernetes.pod.uid'];
const nodeName = fields['kubernetes.node.name'];
const clusterName = fields['orchestrator.cluster.name'];

const pod: Asset = {
'@timestamp': new Date().toISOString(),
'asset.kind': 'pod',
'asset.id': podUid,
'asset.ean': `pod:${podUid}`,
'asset.parents': [`host:${nodeName}`],
};
const pod: Asset = {
'@timestamp': new Date().toISOString(),
'asset.kind': 'pod',
'asset.id': podUid,
'asset.ean': `pod:${podUid}`,
'asset.parents': [`host:${nodeName}`],
};

if (fields['cloud.provider']) {
pod['cloud.provider'] = fields['cloud.provider'];
}
if (fields['cloud.provider']) {
pod['cloud.provider'] = fields['cloud.provider'];
}

if (clusterName) {
pod['orchestrator.cluster.name'] = clusterName;
}
if (clusterName) {
pod['orchestrator.cluster.name'] = clusterName;
}

acc.push(pod);
acc.push(pod);

return acc;
}, []);
return acc;
}, []);

const hitsLen = esResponse.hits.hits.length;
const next = hitsLen === QUERY_MAX_SIZE ? esResponse.hits.hits[hitsLen - 1].sort : undefined;
return { assets, afterKey: next };
});

return result;
const hitsLen = esResponse.hits.hits.length;
const next = hitsLen === QUERY_MAX_SIZE ? esResponse.hits.hits[hitsLen - 1].sort : undefined;
return { assets, afterKey: next };
}
72 changes: 33 additions & 39 deletions x-pack/plugins/asset_manager/server/lib/collectors/services.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,11 @@
import { estypes } from '@elastic/elasticsearch';
import { Asset } from '../../../common/types_api';
import { CollectorOptions, QUERY_MAX_SIZE } from '.';
import { withSpan } from './helpers';

export async function collectServices({
client,
from,
to,
transaction,
sourceIndices,
afterKey,
}: CollectorOptions) {
Expand Down Expand Up @@ -89,49 +87,45 @@ export async function collectServices({

const esResponse = await client.search(dsl);

const result = withSpan({ transaction, name: 'processing_response' }, async () => {
const { after_key: nextKey, buckets = [] } = (esResponse.aggregations?.services || {}) as any;
const assets = buckets.reduce((acc: Asset[], bucket: any) => {
const {
key: { serviceName, serviceEnvironment },
container_and_hosts: containerHosts,
} = bucket;
const { after_key: nextKey, buckets = [] } = (esResponse.aggregations?.services || {}) as any;
const assets = buckets.reduce((acc: Asset[], bucket: any) => {
const {
key: { serviceName, serviceEnvironment },
container_and_hosts: containerHosts,
} = bucket;

if (!serviceName) {
return acc;
}

const service: Asset = {
'@timestamp': new Date().toISOString(),
'asset.kind': 'service',
'asset.id': serviceName,
'asset.ean': `service:${serviceName}`,
'asset.references': [],
'asset.parents': [],
};
if (!serviceName) {
return acc;
}

if (serviceEnvironment) {
service['service.environment'] = serviceEnvironment;
}
const service: Asset = {
'@timestamp': new Date().toISOString(),
'asset.kind': 'service',
'asset.id': serviceName,
'asset.ean': `service:${serviceName}`,
'asset.references': [],
'asset.parents': [],
};

containerHosts.buckets?.forEach((containerBucket: any) => {
const [containerId, hostname] = containerBucket.key;
if (containerId) {
(service['asset.parents'] as string[]).push(`container:${containerId}`);
}
if (serviceEnvironment) {
service['service.environment'] = serviceEnvironment;
}

if (hostname) {
(service['asset.references'] as string[]).push(`host:${hostname}`);
}
});
containerHosts.buckets?.forEach((containerBucket: any) => {
const [containerId, hostname] = containerBucket.key;
if (containerId) {
(service['asset.parents'] as string[]).push(`container:${containerId}`);
}

acc.push(service);
if (hostname) {
(service['asset.references'] as string[]).push(`host:${hostname}`);
}
});

return acc;
}, []);
acc.push(service);

return { assets, afterKey: buckets.length === QUERY_MAX_SIZE ? nextKey : undefined };
});
return acc;
}, []);

return result;
return { assets, afterKey: buckets.length === QUERY_MAX_SIZE ? nextKey : undefined };
}
3 changes: 1 addition & 2 deletions x-pack/plugins/asset_manager/server/lib/implicit_collection/collector_runner.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import apm from 'elastic-apm-node';
import { ImplicitCollectionOptions } from '.';
import { Collector, CollectorOptions, QUERY_MAX_SIZE } from '../collectors';
import { Asset } from '../../../common/types_api';
import { withSpan } from '../collectors/helpers';
import { withSpan } from './helpers';

const TRANSACTION_TYPE = 'asset_manager-implicit_collection';
const transactionName = (collectorName: string) => `asset_manager-collector_${collectorName}`;
Expand All @@ -36,7 +36,6 @@ export class CollectorRunner {
from,
to,
client: this.options.inputClient,
transaction,
sourceIndices: this.options.sourceIndices,
};

Expand Down
0 ..._manager/server/lib/collectors/helpers.ts → ...server/lib/implicit_collection/helpers.ts
View file
Open in desktop
File renamed without changes.