Skip to content

Commit

Permalink
[APM] Service maps - Fix missing ML status for services with jobs but…
Browse files Browse the repository at this point in the history
… no anomalies (elastic#68486)

* Closes elastic#68485 by:
  - use the ML plugin to query for all APM jobs
  - inspect the ml job groups to find all jobs related to a particular service
  - use the mlAnomalySearch client to get ml job buckets with the max anomaly score
  - query for the model_plot buckets to obtain actual/median values for the ML description
  - return the relevant ML job with the max anomaly score for a service
  - indicate to the user that no anomalies were found for a service with an ml job

* - Use the anomalyDetectorsProvider jobs API rather than the search endpoint directly
- Defines a specific return type for the ml jobs api
- Update the empty anomaly data message

* Code and types cleanup

* Return to using record result type on anomaly queries. These are the
same values used in the anomaly explorer and it includes actual &
typical values which greatly improve performance of the previous query.

* - If anomaly data is missing show a gray border around node

* - moved AnomalyDetection out of service map Contents into own component

Co-authored-by: Elastic Machine <[email protected]>
# Conflicts:
#	x-pack/plugins/ml/server/shared.ts
  • Loading branch information
ogupte committed Jun 10, 2020
1 parent 346b61b commit 23f75d0
Show file tree
Hide file tree
Showing 20 changed files with 495 additions and 303 deletions.
2 changes: 2 additions & 0 deletions x-pack/plugins/apm/common/ml_job_constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ export enum severity {
warning = 'warning',
}

export const APM_ML_JOB_GROUP_NAME = 'apm';

export function getMlPrefix(serviceName: string, transactionType?: string) {
const maybeTransactionType = transactionType ? `${transactionType}-` : '';
return encodeForMlApi(`${serviceName}-${maybeTransactionType}`);
Expand Down
10 changes: 10 additions & 0 deletions x-pack/plugins/apm/common/service_map.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,16 @@ export interface Connection {
destination: ConnectionNode;
}

export interface ServiceAnomaly {
anomaly_score: number;
anomaly_severity: string;
actual_value: number;
typical_value: number;
ml_job_id: string;
}

export type ServiceNode = ConnectionNode & Partial<ServiceAnomaly>;

export interface ServiceNodeMetrics {
avgMemoryUsage: number | null;
avgCpuUsage: number | null;
Expand Down
21 changes: 21 additions & 0 deletions x-pack/plugins/apm/common/utils/left_join.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { Assign, Omit } from 'utility-types';

export function leftJoin<
TL extends object,
K extends keyof TL,
TR extends Pick<TL, K>
>(leftRecords: TL[], matchKey: K, rightRecords: TR[]) {
const rightLookup = new Map(
rightRecords.map((record) => [record[matchKey], record])
);
return leftRecords.map((record) => {
const matchProp = (record[matchKey] as unknown) as TR[K];
const matchingRightRecord = rightLookup.get(matchProp);
return { ...record, ...matchingRightRecord };
}) as Array<Assign<TL, Partial<Omit<TR, K>>>>;
}
3 changes: 2 additions & 1 deletion x-pack/plugins/apm/kibana.json
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@
"actions",
"alerting",
"observability",
"security"
"security",
"ml"
],
"server": true,
"ui": true,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,24 +9,15 @@ import {
EuiFlexItem,
EuiHorizontalRule,
EuiTitle,
EuiIconTip,
EuiHealth,
} from '@elastic/eui';
import theme from '@elastic/eui/dist/eui_theme_light.json';
import { i18n } from '@kbn/i18n';
import cytoscape from 'cytoscape';
import React, { MouseEvent } from 'react';
import styled from 'styled-components';
import { fontSize, px } from '../../../../style/variables';
import { Buttons } from './Buttons';
import { Info } from './Info';
import { ServiceMetricFetcher } from './ServiceMetricFetcher';
import { MLJobLink } from '../../../shared/Links/MachineLearningLinks/MLJobLink';
import { getSeverityColor } from '../cytoscapeOptions';
import { asInteger } from '../../../../utils/formatters';
import { getMetricChangeDescription } from '../../../../../../ml/public';

const popoverMinWidth = 280;
import { AnomalyDetection } from './anomaly_detection';
import { ServiceNode } from '../../../../../common/service_map';
import { popoverMinWidth } from '../cytoscapeOptions';

interface ContentsProps {
isService: boolean;
Expand All @@ -36,31 +27,6 @@ interface ContentsProps {
selectedNodeServiceName: string;
}

const HealthStatusTitle = styled(EuiTitle)`
display: inline;
text-transform: uppercase;
`;

const VerticallyCentered = styled.div`
display: flex;
align-items: center;
`;

const SubduedText = styled.span`
color: ${theme.euiTextSubduedColor};
`;

const EnableText = styled.section`
color: ${theme.euiTextSubduedColor};
line-height: 1.4;
font-size: ${fontSize};
width: ${px(popoverMinWidth)};
`;

export const ContentLine = styled.section`
line-height: 2;
`;

// IE 11 does not handle flex properties as expected. With browser detection,
// we can use regular div elements to render contents that are almost identical.
//
Expand All @@ -85,61 +51,13 @@ const FlexColumnGroup = (props: {
const FlexColumnItem = (props: { children: React.ReactNode }) =>
isIE11 ? <div {...props} /> : <EuiFlexItem {...props} />;

const ANOMALY_DETECTION_TITLE = i18n.translate(
'xpack.apm.serviceMap.anomalyDetectionPopoverTitle',
{ defaultMessage: 'Anomaly Detection' }
);

const ANOMALY_DETECTION_TOOLTIP = i18n.translate(
'xpack.apm.serviceMap.anomalyDetectionPopoverTooltip',
{
defaultMessage:
'Service health indicators are powered by the anomaly detection feature in machine learning',
}
);

const ANOMALY_DETECTION_SCORE_METRIC = i18n.translate(
'xpack.apm.serviceMap.anomalyDetectionPopoverScoreMetric',
{ defaultMessage: 'Score (max.)' }
);

const ANOMALY_DETECTION_LINK = i18n.translate(
'xpack.apm.serviceMap.anomalyDetectionPopoverLink',
{ defaultMessage: 'View anomalies' }
);

const ANOMALY_DETECTION_DISABLED_TEXT = i18n.translate(
'xpack.apm.serviceMap.anomalyDetectionPopoverDisabled',
{
defaultMessage:
'Display service health indicators by enabling anomaly detection from the Integrations menu in the Service details view.',
}
);

export function Contents({
selectedNodeData,
isService,
label,
onFocusClick,
selectedNodeServiceName,
}: ContentsProps) {
// Anomaly Detection
const severity = selectedNodeData.severity;
const maxScore = selectedNodeData.max_score;
const actualValue = selectedNodeData.actual_value;
const typicalValue = selectedNodeData.typical_value;
const jobId = selectedNodeData.job_id;
const hasAnomalyDetection = [
severity,
maxScore,
actualValue,
typicalValue,
jobId,
].every((value) => value !== undefined);
const anomalyDescription = hasAnomalyDetection
? getMetricChangeDescription(actualValue, typicalValue).message
: null;

return (
<FlexColumnGroup
direction="column"
Expand All @@ -154,50 +72,7 @@ export function Contents({
</FlexColumnItem>
{isService && (
<FlexColumnItem>
{hasAnomalyDetection ? (
<>
<section>
<HealthStatusTitle size="xxs">
<h3>{ANOMALY_DETECTION_TITLE}</h3>
</HealthStatusTitle>
&nbsp;
<EuiIconTip
type="iInCircle"
content={ANOMALY_DETECTION_TOOLTIP}
/>
</section>
<ContentLine>
<EuiFlexGroup>
<EuiFlexItem>
<VerticallyCentered>
<EuiHealth color={getSeverityColor(severity)} />
<SubduedText>
{ANOMALY_DETECTION_SCORE_METRIC}
</SubduedText>
</VerticallyCentered>
</EuiFlexItem>
<EuiFlexItem grow={false}>
<div>
{asInteger(maxScore)}
<SubduedText>&nbsp;({anomalyDescription})</SubduedText>
</div>
</EuiFlexItem>
</EuiFlexGroup>
</ContentLine>
<ContentLine>
<MLJobLink external jobId={jobId}>
{ANOMALY_DETECTION_LINK}
</MLJobLink>
</ContentLine>
</>
) : (
<>
<HealthStatusTitle size="xxs">
<h3>{ANOMALY_DETECTION_TITLE}</h3>
</HealthStatusTitle>
<EnableText>{ANOMALY_DETECTION_DISABLED_TEXT}</EnableText>
</>
)}
<AnomalyDetection serviceNodeData={selectedNodeData as ServiceNode} />
<EuiHorizontalRule margin="xs" />
</FlexColumnItem>
)}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,156 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

import { i18n } from '@kbn/i18n';
import React from 'react';
import styled from 'styled-components';
import theme from '@elastic/eui/dist/eui_theme_light.json';
import {
EuiFlexGroup,
EuiFlexItem,
EuiTitle,
EuiIconTip,
EuiHealth,
} from '@elastic/eui';
import { fontSize, px } from '../../../../style/variables';
import { asInteger } from '../../../../utils/formatters';
import { MLJobLink } from '../../../shared/Links/MachineLearningLinks/MLJobLink';
import { getSeverityColor, popoverMinWidth } from '../cytoscapeOptions';
import { getMetricChangeDescription } from '../../../../../../ml/public';
import { ServiceNode } from '../../../../../common/service_map';

const HealthStatusTitle = styled(EuiTitle)`
display: inline;
text-transform: uppercase;
`;

const VerticallyCentered = styled.div`
display: flex;
align-items: center;
`;

const SubduedText = styled.span`
color: ${theme.euiTextSubduedColor};
`;

const EnableText = styled.section`
color: ${theme.euiTextSubduedColor};
line-height: 1.4;
font-size: ${fontSize};
width: ${px(popoverMinWidth)};
`;

export const ContentLine = styled.section`
line-height: 2;
`;

interface AnomalyDetectionProps {
serviceNodeData: cytoscape.NodeDataDefinition & ServiceNode;
}

export function AnomalyDetection({ serviceNodeData }: AnomalyDetectionProps) {
const anomalySeverity = serviceNodeData.anomaly_severity;
const anomalyScore = serviceNodeData.anomaly_score;
const actualValue = serviceNodeData.actual_value;
const typicalValue = serviceNodeData.typical_value;
const mlJobId = serviceNodeData.ml_job_id;
const hasAnomalyDetectionScore =
anomalySeverity !== undefined && anomalyScore !== undefined;
const anomalyDescription =
hasAnomalyDetectionScore &&
actualValue !== undefined &&
typicalValue !== undefined
? getMetricChangeDescription(actualValue, typicalValue).message
: null;

return (
<>
<section>
<HealthStatusTitle size="xxs">
<h3>{ANOMALY_DETECTION_TITLE}</h3>
</HealthStatusTitle>
&nbsp;
<EuiIconTip type="iInCircle" content={ANOMALY_DETECTION_TOOLTIP} />
{!mlJobId && <EnableText>{ANOMALY_DETECTION_DISABLED_TEXT}</EnableText>}
</section>
{hasAnomalyDetectionScore && (
<ContentLine>
<EuiFlexGroup>
<EuiFlexItem>
<VerticallyCentered>
<EuiHealth color={getSeverityColor(anomalySeverity)} />
<SubduedText>{ANOMALY_DETECTION_SCORE_METRIC}</SubduedText>
</VerticallyCentered>
</EuiFlexItem>
<EuiFlexItem grow={false}>
<div>
{getDisplayedAnomalyScore(anomalyScore as number)}
{anomalyDescription && (
<SubduedText>&nbsp;({anomalyDescription})</SubduedText>
)}
</div>
</EuiFlexItem>
</EuiFlexGroup>
</ContentLine>
)}
{mlJobId && !hasAnomalyDetectionScore && (
<EnableText>{ANOMALY_DETECTION_NO_DATA_TEXT}</EnableText>
)}
{mlJobId && (
<ContentLine>
<MLJobLink external jobId={mlJobId}>
{ANOMALY_DETECTION_LINK}
</MLJobLink>
</ContentLine>
)}
</>
);
}

function getDisplayedAnomalyScore(score: number) {
if (score > 0 && score < 1) {
return '< 1';
}
return asInteger(score);
}

const ANOMALY_DETECTION_TITLE = i18n.translate(
'xpack.apm.serviceMap.anomalyDetectionPopoverTitle',
{ defaultMessage: 'Anomaly Detection' }
);

const ANOMALY_DETECTION_TOOLTIP = i18n.translate(
'xpack.apm.serviceMap.anomalyDetectionPopoverTooltip',
{
defaultMessage:
'Service health indicators are powered by the anomaly detection feature in machine learning',
}
);

const ANOMALY_DETECTION_SCORE_METRIC = i18n.translate(
'xpack.apm.serviceMap.anomalyDetectionPopoverScoreMetric',
{ defaultMessage: 'Score (max.)' }
);

const ANOMALY_DETECTION_LINK = i18n.translate(
'xpack.apm.serviceMap.anomalyDetectionPopoverLink',
{ defaultMessage: 'View anomalies' }
);

const ANOMALY_DETECTION_DISABLED_TEXT = i18n.translate(
'xpack.apm.serviceMap.anomalyDetectionPopoverDisabled',
{
defaultMessage:
'Display service health indicators by enabling anomaly detection from the Integrations menu in the Service details view.',
}
);

const ANOMALY_DETECTION_NO_DATA_TEXT = i18n.translate(
'xpack.apm.serviceMap.anomalyDetectionPopoverNoData',
{
defaultMessage: `We couldn't find an anomaly score within the selected time range. See details in the anomaly explorer.`,
}
);
Loading

0 comments on commit 23f75d0

Please sign in to comment.