Skip to content

Commit

Permalink
remove sync apm spans
Browse files Browse the repository at this point in the history
  • Loading branch information
klacabane committed Jul 18, 2023
1 parent b527106 commit 20d9fc5
Show file tree
Hide file tree
Showing 7 changed files with 118 additions and 151 deletions.
50 changes: 22 additions & 28 deletions x-pack/plugins/asset_manager/server/lib/collectors/containers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,11 @@
import { estypes } from '@elastic/elasticsearch';
import { Asset } from '../../../common/types_api';
import { CollectorOptions, QUERY_MAX_SIZE } from '.';
import { withSpan } from './helpers';

export async function collectContainers({
client,
from,
to,
transaction,
sourceIndices,
afterKey,
}: CollectorOptions) {
Expand Down Expand Up @@ -62,36 +60,32 @@ export async function collectContainers({

const esResponse = await client.search(dsl);

const result = withSpan({ transaction, name: 'processing_response' }, async () => {
const assets = esResponse.hits.hits.reduce<Asset[]>((acc: Asset[], hit: any) => {
const { fields = {} } = hit;
const containerId = fields['container.id'];
const podUid = fields['kubernetes.pod.uid'];
const nodeName = fields['kubernetes.node.name'];
const assets = esResponse.hits.hits.reduce<Asset[]>((acc: Asset[], hit: any) => {
const { fields = {} } = hit;
const containerId = fields['container.id'];
const podUid = fields['kubernetes.pod.uid'];
const nodeName = fields['kubernetes.node.name'];

const parentEan = podUid ? `pod:${podUid}` : `host:${fields['host.hostname']}`;
const parentEan = podUid ? `pod:${podUid}` : `host:${fields['host.hostname']}`;

const container: Asset = {
'@timestamp': new Date().toISOString(),
'asset.kind': 'container',
'asset.id': containerId,
'asset.ean': `container:${containerId}`,
'asset.parents': [parentEan],
};
const container: Asset = {
'@timestamp': new Date().toISOString(),
'asset.kind': 'container',
'asset.id': containerId,
'asset.ean': `container:${containerId}`,
'asset.parents': [parentEan],
};

if (nodeName) {
container['asset.references'] = [`host:${nodeName}`];
}
if (nodeName) {
container['asset.references'] = [`host:${nodeName}`];
}

acc.push(container);
acc.push(container);

return acc;
}, []);
return acc;
}, []);

const hitsLen = esResponse.hits.hits.length;
const next = hitsLen === QUERY_MAX_SIZE ? esResponse.hits.hits[hitsLen - 1].sort : undefined;
return { assets, afterKey: next };
});

return result;
const hitsLen = esResponse.hits.hits.length;
const next = hitsLen === QUERY_MAX_SIZE ? esResponse.hits.hits[hitsLen - 1].sort : undefined;
return { assets, afterKey: next };
}
80 changes: 37 additions & 43 deletions x-pack/plugins/asset_manager/server/lib/collectors/hosts.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,11 @@
import { estypes } from '@elastic/elasticsearch';
import { Asset } from '../../../common/types_api';
import { CollectorOptions, QUERY_MAX_SIZE } from '.';
import { withSpan } from './helpers';

export async function collectHosts({
client,
from,
to,
transaction,
sourceIndices,
afterKey,
}: CollectorOptions) {
Expand Down Expand Up @@ -61,56 +59,52 @@ export async function collectHosts({

const esResponse = await client.search(dsl);

const result = withSpan({ transaction, name: 'processing_response' }, async () => {
const assets = esResponse.hits.hits.reduce<Asset[]>((acc: Asset[], hit: any) => {
const { fields = {} } = hit;
const hostName = fields['host.hostname'];
const k8sNode = fields['kubernetes.node.name'];
const k8sPod = fields['kubernetes.pod.uid'];
const assets = esResponse.hits.hits.reduce<Asset[]>((acc: Asset[], hit: any) => {
const { fields = {} } = hit;
const hostName = fields['host.hostname'];
const k8sNode = fields['kubernetes.node.name'];
const k8sPod = fields['kubernetes.pod.uid'];

const hostEan = `host:${k8sNode || hostName}`;
const hostEan = `host:${k8sNode || hostName}`;

const host: Asset = {
'@timestamp': new Date().toISOString(),
'asset.kind': 'host',
'asset.id': k8sNode || hostName,
'asset.name': k8sNode || hostName,
'asset.ean': hostEan,
};
const host: Asset = {
'@timestamp': new Date().toISOString(),
'asset.kind': 'host',
'asset.id': k8sNode || hostName,
'asset.name': k8sNode || hostName,
'asset.ean': hostEan,
};

if (fields['cloud.provider']) {
host['cloud.provider'] = fields['cloud.provider'];
}
if (fields['cloud.provider']) {
host['cloud.provider'] = fields['cloud.provider'];
}

if (fields['cloud.instance.id']) {
host['cloud.instance.id'] = fields['cloud.instance.id'];
}
if (fields['cloud.instance.id']) {
host['cloud.instance.id'] = fields['cloud.instance.id'];
}

if (fields['cloud.service.name']) {
host['cloud.service.name'] = fields['cloud.service.name'];
}
if (fields['cloud.service.name']) {
host['cloud.service.name'] = fields['cloud.service.name'];
}

if (fields['cloud.region']) {
host['cloud.region'] = fields['cloud.region'];
}
if (fields['cloud.region']) {
host['cloud.region'] = fields['cloud.region'];
}

if (fields['orchestrator.cluster.name']) {
host['orchestrator.cluster.name'] = fields['orchestrator.cluster.name'];
}
if (fields['orchestrator.cluster.name']) {
host['orchestrator.cluster.name'] = fields['orchestrator.cluster.name'];
}

if (k8sPod) {
host['asset.children'] = [`pod:${k8sPod}`];
}
if (k8sPod) {
host['asset.children'] = [`pod:${k8sPod}`];
}

acc.push(host);
acc.push(host);

return acc;
}, []);
return acc;
}, []);

const hitsLen = esResponse.hits.hits.length;
const next = hitsLen === QUERY_MAX_SIZE ? esResponse.hits.hits[hitsLen - 1].sort : undefined;
return { assets, afterKey: next };
});

return result;
const hitsLen = esResponse.hits.hits.length;
const next = hitsLen === QUERY_MAX_SIZE ? esResponse.hits.hits[hitsLen - 1].sort : undefined;
return { assets, afterKey: next };
}
2 changes: 0 additions & 2 deletions x-pack/plugins/asset_manager/server/lib/collectors/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
* 2.0.
*/

import { Transaction } from 'elastic-apm-node';
import { estypes } from '@elastic/elasticsearch';
import { ElasticsearchClient } from '@kbn/core/server';
import { AssetManagerConfig } from '../../types';
Expand All @@ -19,7 +18,6 @@ export interface CollectorOptions {
client: ElasticsearchClient;
from: number;
to: number;
transaction?: Transaction | null;
sourceIndices: AssetManagerConfig['sourceIndices'];
afterKey?: estypes.SortResults;
}
Expand Down
62 changes: 25 additions & 37 deletions x-pack/plugins/asset_manager/server/lib/collectors/pods.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,16 +8,8 @@
import { estypes } from '@elastic/elasticsearch';
import { Asset } from '../../../common/types_api';
import { CollectorOptions, QUERY_MAX_SIZE } from '.';
import { withSpan } from './helpers';

export async function collectPods({
client,
from,
to,
transaction,
sourceIndices,
afterKey,
}: CollectorOptions) {
export async function collectPods({ client, from, to, sourceIndices, afterKey }: CollectorOptions) {
const { metrics, logs, traces } = sourceIndices;
const dsl: estypes.SearchRequest = {
index: [metrics, logs, traces],
Expand Down Expand Up @@ -60,38 +52,34 @@ export async function collectPods({

const esResponse = await client.search(dsl);

const result = withSpan({ transaction, name: 'processing_response' }, async () => {
const assets = esResponse.hits.hits.reduce<Asset[]>((acc: Asset[], hit: any) => {
const { fields = {} } = hit;
const podUid = fields['kubernetes.pod.uid'];
const nodeName = fields['kubernetes.node.name'];
const clusterName = fields['orchestrator.cluster.name'];
const assets = esResponse.hits.hits.reduce<Asset[]>((acc: Asset[], hit: any) => {
const { fields = {} } = hit;
const podUid = fields['kubernetes.pod.uid'];
const nodeName = fields['kubernetes.node.name'];
const clusterName = fields['orchestrator.cluster.name'];

const pod: Asset = {
'@timestamp': new Date().toISOString(),
'asset.kind': 'pod',
'asset.id': podUid,
'asset.ean': `pod:${podUid}`,
'asset.parents': [`host:${nodeName}`],
};
const pod: Asset = {
'@timestamp': new Date().toISOString(),
'asset.kind': 'pod',
'asset.id': podUid,
'asset.ean': `pod:${podUid}`,
'asset.parents': [`host:${nodeName}`],
};

if (fields['cloud.provider']) {
pod['cloud.provider'] = fields['cloud.provider'];
}
if (fields['cloud.provider']) {
pod['cloud.provider'] = fields['cloud.provider'];
}

if (clusterName) {
pod['orchestrator.cluster.name'] = clusterName;
}
if (clusterName) {
pod['orchestrator.cluster.name'] = clusterName;
}

acc.push(pod);
acc.push(pod);

return acc;
}, []);
return acc;
}, []);

const hitsLen = esResponse.hits.hits.length;
const next = hitsLen === QUERY_MAX_SIZE ? esResponse.hits.hits[hitsLen - 1].sort : undefined;
return { assets, afterKey: next };
});

return result;
const hitsLen = esResponse.hits.hits.length;
const next = hitsLen === QUERY_MAX_SIZE ? esResponse.hits.hits[hitsLen - 1].sort : undefined;
return { assets, afterKey: next };
}
72 changes: 33 additions & 39 deletions x-pack/plugins/asset_manager/server/lib/collectors/services.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,11 @@
import { estypes } from '@elastic/elasticsearch';
import { Asset } from '../../../common/types_api';
import { CollectorOptions, QUERY_MAX_SIZE } from '.';
import { withSpan } from './helpers';

export async function collectServices({
client,
from,
to,
transaction,
sourceIndices,
afterKey,
}: CollectorOptions) {
Expand Down Expand Up @@ -89,49 +87,45 @@ export async function collectServices({

const esResponse = await client.search(dsl);

const result = withSpan({ transaction, name: 'processing_response' }, async () => {
const { after_key: nextKey, buckets = [] } = (esResponse.aggregations?.services || {}) as any;
const assets = buckets.reduce((acc: Asset[], bucket: any) => {
const {
key: { serviceName, serviceEnvironment },
container_and_hosts: containerHosts,
} = bucket;
const { after_key: nextKey, buckets = [] } = (esResponse.aggregations?.services || {}) as any;
const assets = buckets.reduce((acc: Asset[], bucket: any) => {
const {
key: { serviceName, serviceEnvironment },
container_and_hosts: containerHosts,
} = bucket;

if (!serviceName) {
return acc;
}

const service: Asset = {
'@timestamp': new Date().toISOString(),
'asset.kind': 'service',
'asset.id': serviceName,
'asset.ean': `service:${serviceName}`,
'asset.references': [],
'asset.parents': [],
};
if (!serviceName) {
return acc;
}

if (serviceEnvironment) {
service['service.environment'] = serviceEnvironment;
}
const service: Asset = {
'@timestamp': new Date().toISOString(),
'asset.kind': 'service',
'asset.id': serviceName,
'asset.ean': `service:${serviceName}`,
'asset.references': [],
'asset.parents': [],
};

containerHosts.buckets?.forEach((containerBucket: any) => {
const [containerId, hostname] = containerBucket.key;
if (containerId) {
(service['asset.parents'] as string[]).push(`container:${containerId}`);
}
if (serviceEnvironment) {
service['service.environment'] = serviceEnvironment;
}

if (hostname) {
(service['asset.references'] as string[]).push(`host:${hostname}`);
}
});
containerHosts.buckets?.forEach((containerBucket: any) => {
const [containerId, hostname] = containerBucket.key;
if (containerId) {
(service['asset.parents'] as string[]).push(`container:${containerId}`);
}

acc.push(service);
if (hostname) {
(service['asset.references'] as string[]).push(`host:${hostname}`);
}
});

return acc;
}, []);
acc.push(service);

return { assets, afterKey: buckets.length === QUERY_MAX_SIZE ? nextKey : undefined };
});
return acc;
}, []);

return result;
return { assets, afterKey: buckets.length === QUERY_MAX_SIZE ? nextKey : undefined };
}
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import apm from 'elastic-apm-node';
import { ImplicitCollectionOptions } from '.';
import { Collector, CollectorOptions, QUERY_MAX_SIZE } from '../collectors';
import { Asset } from '../../../common/types_api';
import { withSpan } from '../collectors/helpers';
import { withSpan } from './helpers';

const TRANSACTION_TYPE = 'asset_manager-implicit_collection';
const transactionName = (collectorName: string) => `asset_manager-collector_${collectorName}`;
Expand All @@ -36,7 +36,6 @@ export class CollectorRunner {
from,
to,
client: this.options.inputClient,
transaction,
sourceIndices: this.options.sourceIndices,
};

Expand Down

0 comments on commit 20d9fc5

Please sign in to comment.