From 48146b8afb05d08836a8c2d19107268475b7c56a Mon Sep 17 00:00:00 2001 From: Hector Hernandez <39923391+hectorhdzg@users.noreply.github.com> Date: Thu, 14 Mar 2024 09:32:05 -0700 Subject: [PATCH] [Azure Monitor OpenTelemetry] Live Metrics updates (#28912) ### Packages impacted by this PR @azure/monitor-opentelemetry Fixed issue with quickpulse document duration Fixed issue with miscalculation in dependency duration metric Updated default quickpulse endpoint --- .../src/metrics/quickpulse/export/sender.ts | 29 +++-- .../src/metrics/quickpulse/liveMetrics.ts | 8 +- .../src/metrics/quickpulse/utils.ts | 116 ++++++++++-------- .../monitor-opentelemetry/src/types.ts | 2 +- .../internal/unit/metrics/liveMetrics.test.ts | 8 +- 5 files changed, 95 insertions(+), 68 deletions(-) diff --git a/sdk/monitor/monitor-opentelemetry/src/metrics/quickpulse/export/sender.ts b/sdk/monitor/monitor-opentelemetry/src/metrics/quickpulse/export/sender.ts index d89d09a757c5..8a5d13170278 100644 --- a/sdk/monitor/monitor-opentelemetry/src/metrics/quickpulse/export/sender.ts +++ b/sdk/monitor/monitor-opentelemetry/src/metrics/quickpulse/export/sender.ts @@ -1,7 +1,9 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. import url from "url"; -import { redirectPolicyName } from "@azure/core-rest-pipeline"; +import { RestError, redirectPolicyName } from "@azure/core-rest-pipeline"; +import { TokenCredential } from "@azure/core-auth"; +import { diag } from "@opentelemetry/api"; import { PingOptionalParams, PingResponse, @@ -10,7 +12,6 @@ import { QuickpulseClient, QuickpulseClientOptionalParams, } from "../../../generated"; -import { TokenCredential } from "@azure/core-auth"; const applicationInsightsResource = "https://monitor.azure.com//.default"; @@ -56,18 +57,30 @@ export class QuickpulseSender { * Ping Quickpulse service * @internal */ - async ping(optionalParams: PingOptionalParams): Promise { - let response = await this.quickpulseClient.ping(this.instrumentationKey, optionalParams); - return response; + async ping(optionalParams: PingOptionalParams): Promise { + try { + let response = await this.quickpulseClient.ping(this.instrumentationKey, optionalParams); + return response; + } catch (error: any) { + const restError = error as RestError; + diag.info("Failed to ping Quickpulse service", restError.message); + } + return; } /** * Post Quickpulse service * @internal */ - async post(optionalParams: PostOptionalParams): Promise { - let response = await this.quickpulseClient.post(this.instrumentationKey, optionalParams); - return response; + async post(optionalParams: PostOptionalParams): Promise { + try { + let response = await this.quickpulseClient.post(this.instrumentationKey, optionalParams); + return response; + } catch (error: any) { + const restError = error as RestError; + diag.warn("Failed to post Quickpulse service", restError.message); + } + return; } handlePermanentRedirect(location: string | undefined) { diff --git a/sdk/monitor/monitor-opentelemetry/src/metrics/quickpulse/liveMetrics.ts b/sdk/monitor/monitor-opentelemetry/src/metrics/quickpulse/liveMetrics.ts index 00349dc13e74..8ac5e9baabc1 100644 --- a/sdk/monitor/monitor-opentelemetry/src/metrics/quickpulse/liveMetrics.ts +++ b/sdk/monitor/monitor-opentelemetry/src/metrics/quickpulse/liveMetrics.ts @@ -42,7 +42,7 @@ import { import { QuickpulseMetricExporter } from "./export/exporter"; import { QuickpulseSender } from "./export/sender"; import { ConnectionStringParser } from "../../utils/connectionStringParser"; -import { DEFAULT_BREEZE_ENDPOINT, DEFAULT_LIVEMETRICS_ENDPOINT } from "../../types"; +import { DEFAULT_LIVEMETRICS_ENDPOINT } from "../../types"; import { QuickPulseOpenTelemetryMetricNames, QuickpulseExporterOptions } from "./types"; import { hrTimeToMilliseconds, suppressTracing } from "@opentelemetry/core"; @@ -137,11 +137,11 @@ export class LiveMetrics { ); this.pingSender = new QuickpulseSender({ endpointUrl: parsedConnectionString.liveendpoint || DEFAULT_LIVEMETRICS_ENDPOINT, - instrumentationKey: parsedConnectionString.instrumentationkey || DEFAULT_BREEZE_ENDPOINT, + instrumentationKey: parsedConnectionString.instrumentationkey || "", }); let exporterOptions: QuickpulseExporterOptions = { endpointUrl: parsedConnectionString.liveendpoint || DEFAULT_LIVEMETRICS_ENDPOINT, - instrumentationKey: parsedConnectionString.instrumentationkey || DEFAULT_BREEZE_ENDPOINT, + instrumentationKey: parsedConnectionString.instrumentationkey || "", postCallback: this.quickPulseDone.bind(this), getDocumentsFn: this.getDocuments.bind(this), baseMonitoringDataPoint: this.baseMonitoringDataPoint, @@ -464,7 +464,7 @@ export class LiveMetrics { } this.lastDependencyDuration = { count: this.totalDependencyCount, - duration: this.requestDuration, + duration: this.dependencyDuration, time: currentTime, }; } diff --git a/sdk/monitor/monitor-opentelemetry/src/metrics/quickpulse/utils.ts b/sdk/monitor/monitor-opentelemetry/src/metrics/quickpulse/utils.ts index c0ca98d1975e..8f0a2c6a172d 100644 --- a/sdk/monitor/monitor-opentelemetry/src/metrics/quickpulse/utils.ts +++ b/sdk/monitor/monitor-opentelemetry/src/metrics/quickpulse/utils.ts @@ -16,8 +16,29 @@ import { } from "../../generated"; import { Attributes, SpanKind } from "@opentelemetry/api"; import { - SemanticAttributes, - SemanticResourceAttributes, + SEMATTRS_EXCEPTION_MESSAGE, + SEMATTRS_EXCEPTION_TYPE, + SEMATTRS_HTTP_HOST, + SEMATTRS_HTTP_METHOD, + SEMATTRS_HTTP_SCHEME, + SEMATTRS_HTTP_STATUS_CODE, + SEMATTRS_HTTP_TARGET, + SEMATTRS_HTTP_URL, + SEMATTRS_NET_PEER_IP, + SEMATTRS_NET_PEER_NAME, + SEMATTRS_NET_PEER_PORT, + SEMATTRS_RPC_GRPC_STATUS_CODE, + SEMRESATTRS_K8S_CRONJOB_NAME, + SEMRESATTRS_K8S_DAEMONSET_NAME, + SEMRESATTRS_K8S_DEPLOYMENT_NAME, + SEMRESATTRS_K8S_JOB_NAME, + SEMRESATTRS_K8S_POD_NAME, + SEMRESATTRS_K8S_REPLICASET_NAME, + SEMRESATTRS_K8S_STATEFULSET_NAME, + SEMRESATTRS_SERVICE_INSTANCE_ID, + SEMRESATTRS_SERVICE_NAME, + SEMRESATTRS_SERVICE_NAMESPACE, + SEMRESATTRS_TELEMETRY_SDK_VERSION, } from "@opentelemetry/semantic-conventions"; import { SDK_INFO, hrTimeToMilliseconds } from "@opentelemetry/core"; import { DataPointType, Histogram, ResourceMetrics } from "@opentelemetry/sdk-metrics"; @@ -36,7 +57,7 @@ import { LogAttributes } from "@opentelemetry/api-logs"; /** Get the internal SDK version */ export function getSdkVersion(): string { const { nodeVersion } = process.versions; - const opentelemetryVersion = SDK_INFO[SemanticResourceAttributes.TELEMETRY_SDK_VERSION]; + const opentelemetryVersion = SDK_INFO[SEMRESATTRS_TELEMETRY_SDK_VERSION]; const version = `ext${AZURE_MONITOR_OPENTELEMETRY_VERSION}`; const internalSdkVersion = `${process.env[AZURE_MONITOR_PREFIX] ?? ""}node${nodeVersion}:otel${opentelemetryVersion}:${version}`; return internalSdkVersion; @@ -57,8 +78,8 @@ export function setSdkPrefix(): void { export function getCloudRole(resource: Resource): string { let cloudRole = ""; // Service attributes - const serviceName = resource.attributes[SemanticResourceAttributes.SERVICE_NAME]; - const serviceNamespace = resource.attributes[SemanticResourceAttributes.SERVICE_NAMESPACE]; + const serviceName = resource.attributes[SEMRESATTRS_SERVICE_NAME]; + const serviceNamespace = resource.attributes[SEMRESATTRS_SERVICE_NAMESPACE]; if (serviceName) { // Custom Service name provided by customer is highest precedence if (!String(serviceName).startsWith("unknown_service")) { @@ -77,31 +98,27 @@ export function getCloudRole(resource: Resource): string { } } // Kubernetes attributes should take precedence - const kubernetesDeploymentName = - resource.attributes[SemanticResourceAttributes.K8S_DEPLOYMENT_NAME]; + const kubernetesDeploymentName = resource.attributes[SEMRESATTRS_K8S_DEPLOYMENT_NAME]; if (kubernetesDeploymentName) { return String(kubernetesDeploymentName); } - const kuberneteReplicasetName = - resource.attributes[SemanticResourceAttributes.K8S_REPLICASET_NAME]; + const kuberneteReplicasetName = resource.attributes[SEMRESATTRS_K8S_REPLICASET_NAME]; if (kuberneteReplicasetName) { return String(kuberneteReplicasetName); } - const kubernetesStatefulSetName = - resource.attributes[SemanticResourceAttributes.K8S_STATEFULSET_NAME]; + const kubernetesStatefulSetName = resource.attributes[SEMRESATTRS_K8S_STATEFULSET_NAME]; if (kubernetesStatefulSetName) { return String(kubernetesStatefulSetName); } - const kubernetesJobName = resource.attributes[SemanticResourceAttributes.K8S_JOB_NAME]; + const kubernetesJobName = resource.attributes[SEMRESATTRS_K8S_JOB_NAME]; if (kubernetesJobName) { return String(kubernetesJobName); } - const kubernetesCronjobName = resource.attributes[SemanticResourceAttributes.K8S_CRONJOB_NAME]; + const kubernetesCronjobName = resource.attributes[SEMRESATTRS_K8S_CRONJOB_NAME]; if (kubernetesCronjobName) { return String(kubernetesCronjobName); } - const kubernetesDaemonsetName = - resource.attributes[SemanticResourceAttributes.K8S_DAEMONSET_NAME]; + const kubernetesDaemonsetName = resource.attributes[SEMRESATTRS_K8S_DAEMONSET_NAME]; if (kubernetesDaemonsetName) { return String(kubernetesDaemonsetName); } @@ -110,12 +127,12 @@ export function getCloudRole(resource: Resource): string { export function getCloudRoleInstance(resource: Resource): string { // Kubernetes attributes should take precedence - const kubernetesPodName = resource.attributes[SemanticResourceAttributes.K8S_POD_NAME]; + const kubernetesPodName = resource.attributes[SEMRESATTRS_K8S_POD_NAME]; if (kubernetesPodName) { return String(kubernetesPodName); } // Service attributes - const serviceInstanceId = resource.attributes[SemanticResourceAttributes.SERVICE_INSTANCE_ID]; + const serviceInstanceId = resource.attributes[SEMRESATTRS_SERVICE_INSTANCE_ID]; if (serviceInstanceId) { return String(serviceInstanceId); } @@ -190,18 +207,23 @@ export function resourceMetricsToQuickpulseDataPoint( return [quickpulseDataPoint]; } +function getIso8601Duration(milliseconds: number) { + const seconds = milliseconds / 1000; + return `PT${seconds}S`; +} + export function getSpanDocument(span: ReadableSpan): Request | RemoteDependency { let document: Request | RemoteDependency = { documentType: KnownDocumentIngressDocumentType.Request, }; - const httpMethod = span.attributes[SemanticAttributes.HTTP_METHOD]; - const grpcStatusCode = span.attributes[SemanticAttributes.RPC_GRPC_STATUS_CODE]; + const httpMethod = span.attributes[SEMATTRS_HTTP_METHOD]; + const grpcStatusCode = span.attributes[SEMATTRS_RPC_GRPC_STATUS_CODE]; let url = ""; let code = ""; if (span.kind === SpanKind.SERVER || span.kind === SpanKind.CONSUMER) { if (httpMethod) { url = getUrl(span.attributes); - const httpStatusCode = span.attributes[SemanticAttributes.HTTP_STATUS_CODE]; + const httpStatusCode = span.attributes[SEMATTRS_HTTP_STATUS_CODE]; if (httpStatusCode) { code = String(httpStatusCode); } @@ -214,11 +236,11 @@ export function getSpanDocument(span: ReadableSpan): Request | RemoteDependency name: span.name, url: url, responseCode: code, - duration: hrTimeToMilliseconds(span.duration).toString(), + duration: getIso8601Duration(hrTimeToMilliseconds(span.duration)), }; } else { url = getUrl(span.attributes); - const httpStatusCode = span.attributes[SemanticAttributes.HTTP_STATUS_CODE]; + const httpStatusCode = span.attributes[SEMATTRS_HTTP_STATUS_CODE]; if (httpStatusCode) { code = String(httpStatusCode); } @@ -228,7 +250,7 @@ export function getSpanDocument(span: ReadableSpan): Request | RemoteDependency name: span.name, commandName: url, resultCode: code, - duration: hrTimeToMilliseconds(span.duration).toString(), + duration: getIso8601Duration(hrTimeToMilliseconds(span.duration)), }; } document.properties = createPropertiesFromAttributes(span.attributes); @@ -239,9 +261,9 @@ export function getLogDocument(logRecord: LogRecord): Trace | Exception { let document: Trace | Exception = { documentType: KnownDocumentIngressDocumentType.Exception, }; - const exceptionType = String(logRecord.attributes[SemanticAttributes.EXCEPTION_TYPE]); + const exceptionType = String(logRecord.attributes[SEMATTRS_EXCEPTION_TYPE]); if (exceptionType) { - const exceptionMessage = String(logRecord.attributes[SemanticAttributes.EXCEPTION_MESSAGE]); + const exceptionMessage = String(logRecord.attributes[SEMATTRS_EXCEPTION_MESSAGE]); document = { documentType: KnownDocumentIngressDocumentType.Exception, exceptionType: exceptionType, @@ -267,26 +289,18 @@ function createPropertiesFromAttributes( if ( !( key.startsWith("_MS.") || - key === SemanticAttributes.NET_PEER_IP || - key === SemanticAttributes.NET_PEER_NAME || - key === SemanticAttributes.PEER_SERVICE || - key === SemanticAttributes.HTTP_METHOD || - key === SemanticAttributes.HTTP_URL || - key === SemanticAttributes.HTTP_STATUS_CODE || - key === SemanticAttributes.HTTP_ROUTE || - key === SemanticAttributes.HTTP_HOST || - key === SemanticAttributes.HTTP_URL || - key === SemanticAttributes.DB_SYSTEM || - key === SemanticAttributes.DB_STATEMENT || - key === SemanticAttributes.DB_OPERATION || - key === SemanticAttributes.DB_NAME || - key === SemanticAttributes.RPC_SYSTEM || - key === SemanticAttributes.RPC_GRPC_STATUS_CODE || - key === SemanticAttributes.EXCEPTION_TYPE || - key === SemanticAttributes.EXCEPTION_MESSAGE + key === SEMATTRS_NET_PEER_IP || + key === SEMATTRS_NET_PEER_NAME || + key === SEMATTRS_HTTP_METHOD || + key === SEMATTRS_HTTP_URL || + key === SEMATTRS_HTTP_STATUS_CODE || + key === SEMATTRS_HTTP_HOST || + key === SEMATTRS_HTTP_URL || + key === SEMATTRS_EXCEPTION_TYPE || + key === SEMATTRS_EXCEPTION_MESSAGE ) ) { - properties.push({ key: key, value: attributes[key] as string }); + properties.push({ key: key, value: String(attributes[key]) }); } } } @@ -297,26 +311,26 @@ function getUrl(attributes: Attributes): string { if (!attributes) { return ""; } - const httpMethod = attributes[SemanticAttributes.HTTP_METHOD]; + const httpMethod = attributes[SEMATTRS_HTTP_METHOD]; if (httpMethod) { - const httpUrl = attributes[SemanticAttributes.HTTP_URL]; + const httpUrl = attributes[SEMATTRS_HTTP_URL]; if (httpUrl) { return String(httpUrl); } else { - const httpScheme = attributes[SemanticAttributes.HTTP_SCHEME]; - const httpTarget = attributes[SemanticAttributes.HTTP_TARGET]; + const httpScheme = attributes[SEMATTRS_HTTP_SCHEME]; + const httpTarget = attributes[SEMATTRS_HTTP_TARGET]; if (httpScheme && httpTarget) { - const httpHost = attributes[SemanticAttributes.HTTP_HOST]; + const httpHost = attributes[SEMATTRS_HTTP_HOST]; if (httpHost) { return `${httpScheme}://${httpHost}${httpTarget}`; } else { - const netPeerPort = attributes[SemanticAttributes.NET_PEER_PORT]; + const netPeerPort = attributes[SEMATTRS_NET_PEER_PORT]; if (netPeerPort) { - const netPeerName = attributes[SemanticAttributes.NET_PEER_NAME]; + const netPeerName = attributes[SEMATTRS_NET_PEER_NAME]; if (netPeerName) { return `${httpScheme}://${netPeerName}:${netPeerPort}${httpTarget}`; } else { - const netPeerIp = attributes[SemanticAttributes.NET_PEER_IP]; + const netPeerIp = attributes[SEMATTRS_NET_PEER_IP]; if (netPeerIp) { return `${httpScheme}://${netPeerIp}:${netPeerPort}${httpTarget}`; } diff --git a/sdk/monitor/monitor-opentelemetry/src/types.ts b/sdk/monitor/monitor-opentelemetry/src/types.ts index 8a23dd8503ba..cc51e09f751e 100644 --- a/sdk/monitor/monitor-opentelemetry/src/types.ts +++ b/sdk/monitor/monitor-opentelemetry/src/types.ts @@ -90,7 +90,7 @@ export const DEFAULT_BREEZE_ENDPOINT = "https://dc.services.visualstudio.com"; * Default Live Metrics endpoint. * @internal */ -export const DEFAULT_LIVEMETRICS_ENDPOINT = "https://rt.services.visualstudio.com"; +export const DEFAULT_LIVEMETRICS_ENDPOINT = "https://global.livediagnostics.monitor.azure.com"; export enum StatsbeatFeature { NONE = 0, diff --git a/sdk/monitor/monitor-opentelemetry/test/internal/unit/metrics/liveMetrics.test.ts b/sdk/monitor/monitor-opentelemetry/test/internal/unit/metrics/liveMetrics.test.ts index 2f2491a20f13..1e715b9e0ea3 100644 --- a/sdk/monitor/monitor-opentelemetry/test/internal/unit/metrics/liveMetrics.test.ts +++ b/sdk/monitor/monitor-opentelemetry/test/internal/unit/metrics/liveMetrics.test.ts @@ -201,13 +201,13 @@ describe("#LiveMetrics", () => { assert.strictEqual(documents[6].documentType, "RemoteDependency"); assert.strictEqual((documents[6] as RemoteDependency).commandName, "http://test.com"); assert.strictEqual((documents[6] as RemoteDependency).resultCode, "200"); - assert.strictEqual((documents[6] as RemoteDependency).duration, "12345678"); + assert.strictEqual((documents[6] as RemoteDependency).duration, "PT12345.678S"); assert.equal((documents[6].properties as any)[0].key, "customAttribute"); assert.equal((documents[6].properties as any)[0].value, "test"); for (let i = 7; i < 9; i++) { assert.strictEqual((documents[i] as Request).url, "http://test.com"); assert.strictEqual((documents[i] as Request).responseCode, "200"); - assert.strictEqual((documents[i] as Request).duration, "98765432"); + assert.strictEqual((documents[i] as Request).duration, "PT98765.432S"); assert.equal((documents[i].properties as any)[0].key, "customAttribute"); assert.equal((documents[i].properties as any)[0].value, "test"); } @@ -215,14 +215,14 @@ describe("#LiveMetrics", () => { assert.strictEqual(documents[i].documentType, "RemoteDependency"); assert.strictEqual((documents[i] as RemoteDependency).commandName, "http://test.com"); assert.strictEqual((documents[i] as RemoteDependency).resultCode, "400"); - assert.strictEqual((documents[i] as RemoteDependency).duration, "900000"); + assert.strictEqual((documents[i] as RemoteDependency).duration, "PT900S"); assert.equal((documents[i].properties as any)[0].key, "customAttribute"); assert.equal((documents[i].properties as any)[0].value, "test"); } for (let i = 12; i < 15; i++) { assert.strictEqual((documents[i] as Request).url, "http://test.com"); assert.strictEqual((documents[i] as Request).responseCode, "400"); - assert.strictEqual((documents[i] as Request).duration, "100000"); + assert.strictEqual((documents[i] as Request).duration, "PT100S"); assert.equal((documents[i].properties as any)[0].key, "customAttribute"); assert.equal((documents[i].properties as any)[0].value, "test"); }