From dd563c72fd4b81c07184b0a4e82c80d6bd1abedd Mon Sep 17 00:00:00 2001 From: Timothy Mothra Date: Wed, 15 Nov 2023 14:41:15 -0800 Subject: [PATCH] [AzureMonitorLiveMetrics] POC (#40001) * working POC * update public api --- ...penTelemetry.LiveMetrics.netstandard2.0.cs | 19 ++ ...e.Monitor.OpenTelemetry.LiveMetrics.csproj | 1 + .../QuickPulseSDKClientAPIsRestClient.cs | 116 ++++++++++ .../LiveMetricsExporterEventSource.cs | 3 + .../src/Internals/Manager.Metrics.cs | 68 ++++++ .../src/Internals/Manager.cs | 207 ++++++++++++++++++ .../src/Internals/ManagerFactory.cs | 36 +++ .../src/LiveMetricConstants.cs | 36 +++ .../src/LiveMetricsExporterOptions.cs | 9 +- .../src/LiveMetricsExtensions.cs | 2 +- .../src/LiveMetricsExtractionProcessor.cs | 30 ++- .../src/LiveMetricsMetricExporter.cs | 81 +++++++ .../src/LiveMetricsTraceExporter.cs | 35 +++ .../Program.cs | 80 ++++++- 14 files changed, 708 insertions(+), 15 deletions(-) create mode 100644 sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/Customizations/QuickPulseSDKClientAPIsRestClient.cs create mode 100644 sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/Internals/Manager.Metrics.cs create mode 100644 sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/Internals/Manager.cs create mode 100644 sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/Internals/ManagerFactory.cs create mode 100644 sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/LiveMetricsMetricExporter.cs create mode 100644 sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/LiveMetricsTraceExporter.cs diff --git a/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/api/Azure.Monitor.OpenTelemetry.LiveMetrics.netstandard2.0.cs b/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/api/Azure.Monitor.OpenTelemetry.LiveMetrics.netstandard2.0.cs index d8a96c69ccf2f..b892927386428 100644 --- a/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/api/Azure.Monitor.OpenTelemetry.LiveMetrics.netstandard2.0.cs +++ b/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/api/Azure.Monitor.OpenTelemetry.LiveMetrics.netstandard2.0.cs @@ -1,3 +1,22 @@ +namespace Azure.Monitor.OpenTelemetry.LiveMetrics +{ + public partial class LiveMetricsExporterOptions : Azure.Core.ClientOptions + { + public LiveMetricsExporterOptions() { } + public string ConnectionString { get { throw null; } set { } } + public Azure.Core.TokenCredential Credential { get { throw null; } set { } } + public bool EnableLiveMetrics { get { throw null; } set { } } + } + public static partial class LiveMetricsExtensions + { + public static OpenTelemetry.Trace.TracerProviderBuilder AddLiveMetrics(this OpenTelemetry.Trace.TracerProviderBuilder builder, System.Action configure = null, string name = null) { throw null; } + } + public partial class LiveMetricsTraceExporter : OpenTelemetry.BaseExporter + { + public LiveMetricsTraceExporter(Azure.Monitor.OpenTelemetry.LiveMetrics.LiveMetricsExporterOptions options) { } + public override OpenTelemetry.ExportResult Export(in OpenTelemetry.Batch batch) { throw null; } + } +} namespace Azure.Monitor.OpenTelemetry.LiveMetrics.Models { [System.Runtime.InteropServices.StructLayoutAttribute(System.Runtime.InteropServices.LayoutKind.Sequential)] diff --git a/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/Azure.Monitor.OpenTelemetry.LiveMetrics.csproj b/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/Azure.Monitor.OpenTelemetry.LiveMetrics.csproj index 048265ac58eed..a7573b152b8cd 100644 --- a/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/Azure.Monitor.OpenTelemetry.LiveMetrics.csproj +++ b/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/Azure.Monitor.OpenTelemetry.LiveMetrics.csproj @@ -14,6 +14,7 @@ + diff --git a/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/Customizations/QuickPulseSDKClientAPIsRestClient.cs b/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/Customizations/QuickPulseSDKClientAPIsRestClient.cs new file mode 100644 index 0000000000000..d7788cfd9c857 --- /dev/null +++ b/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/Customizations/QuickPulseSDKClientAPIsRestClient.cs @@ -0,0 +1,116 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#nullable disable + +using System; +using System.Collections.Generic; +using System.Text.Json; +using System.Threading; +using Azure.Core; +using Azure.Monitor.OpenTelemetry.LiveMetrics.Models; + +namespace Azure.Monitor.OpenTelemetry.LiveMetrics +{ + internal partial class QuickPulseSDKClientAPIsRestClient + { + /// SDK ping. + /// The ikey of the target Application Insights component that displays server info sent by /QuickPulseService.svc/ping. + /// Deprecated. An alternative way to pass api key. Use AAD auth instead. + /// Timestamp when SDK transmits the metrics and documents to QuickPulse. A 8-byte long type of ticks. + /// Computer name where AI SDK lives. QuickPulse uses machine name with instance name as a backup. + /// Service instance name where AI SDK lives. QuickPulse uses machine name with instance name as a backup. + /// Identifies an AI SDK as trusted agent to report metrics and documents. + /// Cloud role name for which SDK reports metrics and documents. + /// Version/generation of the data contract (MonitoringDataPoint) between SDK and QuickPulse. + /// An encoded string that indicates whether the collection configuration is changed. + /// Data contract between SDK and QuickPulse. /QuickPulseService.svc/ping uses this as a backup source of machine name, instance name and invariant version. + /// The cancellation token to use. + /// is null. + public ResponseWithHeaders PingCustom(string ikey, string apikey = null, int? xMsQpsTransmissionTime = null, string xMsQpsMachineName = null, string xMsQpsInstanceName = null, string xMsQpsStreamId = null, string xMsQpsRoleName = null, string xMsQpsInvariantVersion = null, string xMsQpsConfigurationEtag = null, MonitoringDataPoint monitoringDataPoint = null, CancellationToken cancellationToken = default) + { + if (ikey == null) + { + throw new ArgumentNullException(nameof(ikey)); + } + + using var message = CreatePingRequest(ikey, apikey, xMsQpsTransmissionTime, xMsQpsMachineName, xMsQpsInstanceName, xMsQpsStreamId, xMsQpsRoleName, xMsQpsInvariantVersion, xMsQpsConfigurationEtag, monitoringDataPoint); + _pipeline.Send(message, cancellationToken); + var headers = new QuickPulseSDKClientAPIsPingHeaders(message.Response); + switch (message.Response.Status) + { + case 200: + { + CollectionConfigurationInfo value = default; + if (message.Response.Headers.ContentLength != 0) + { + using var document = JsonDocument.Parse(message.Response.ContentStream); + value = CollectionConfigurationInfo.DeserializeCollectionConfigurationInfo(document.RootElement); + } + return ResponseWithHeaders.FromValue(value, headers, message.Response); + } + case 400: + case 401: + case 403: + case 404: + case 500: + case 503: + { + ServiceError value = default; + using var document = JsonDocument.Parse(message.Response.ContentStream); + value = ServiceError.DeserializeServiceError(document.RootElement); + return ResponseWithHeaders.FromValue(value, headers, message.Response); + } + default: + throw new RequestFailedException(message.Response); + } + } + + /// SDK post. + /// The ikey of the target Application Insights component that displays metrics and documents sent by /QuickPulseService.svc/post. + /// An alternative way to pass api key. Deprecated. Use AAD authentication instead. + /// An encoded string that indicates whether the collection configuration is changed. + /// Timestamp when SDK transmits the metrics and documents to QuickPulse. A 8-byte long type of ticks. + /// Data contract between SDK and QuickPulse. /QuickPulseService.svc/post uses this to publish metrics and documents to the backend QuickPulse server. + /// The cancellation token to use. + /// is null. + public ResponseWithHeaders PostCustom(string ikey, string apikey = null, string xMsQpsConfigurationEtag = null, int? xMsQpsTransmissionTime = null, IEnumerable monitoringDataPoints = null, CancellationToken cancellationToken = default) + { + if (ikey == null) + { + throw new ArgumentNullException(nameof(ikey)); + } + + using var message = CreatePostRequest(ikey, apikey, xMsQpsConfigurationEtag, xMsQpsTransmissionTime, monitoringDataPoints); + _pipeline.Send(message, cancellationToken); + var headers = new QuickPulseSDKClientAPIsPostHeaders(message.Response); + switch (message.Response.Status) + { + case 200: + { + CollectionConfigurationInfo value = default; + if (message.Response.Headers.ContentLength != 0) + { + using var document = JsonDocument.Parse(message.Response.ContentStream); + value = CollectionConfigurationInfo.DeserializeCollectionConfigurationInfo(document.RootElement); + } + return ResponseWithHeaders.FromValue(value, headers, message.Response); + } + case 400: + case 401: + case 403: + case 404: + case 500: + case 503: + { + ServiceError value = default; + using var document = JsonDocument.Parse(message.Response.ContentStream); + value = ServiceError.DeserializeServiceError(document.RootElement); + return ResponseWithHeaders.FromValue(value, headers, message.Response); + } + default: + throw new RequestFailedException(message.Response); + } + } + } +} diff --git a/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/Internals/Diagnostics/LiveMetricsExporterEventSource.cs b/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/Internals/Diagnostics/LiveMetricsExporterEventSource.cs index d1219903715af..a41675dc234bc 100644 --- a/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/Internals/Diagnostics/LiveMetricsExporterEventSource.cs +++ b/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/Internals/Diagnostics/LiveMetricsExporterEventSource.cs @@ -104,5 +104,8 @@ public void ErrorInitializingPartOfSdkVersion(string typeName, Exception ex) [Event(6, Message = "Failed to get Type version while initialize SDK version due to an exception. Not user actionable. Type: {0}. {1}", Level = EventLevel.Warning)] public void ErrorInitializingPartOfSdkVersion(string typeName, string exceptionMessage) => WriteEvent(6, typeName, exceptionMessage); + + [Event(7, Message = "HttpPipelineBuilder is built with AAD Credentials. TokenCredential: {0} Scope: {1}", Level = EventLevel.Informational)] + public void SetAADCredentialsToPipeline(string credentialTypeName, string scope) => WriteEvent(7, credentialTypeName, scope); } } diff --git a/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/Internals/Manager.Metrics.cs b/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/Internals/Manager.Metrics.cs new file mode 100644 index 0000000000000..dce27d5b09c4f --- /dev/null +++ b/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/Internals/Manager.Metrics.cs @@ -0,0 +1,68 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Diagnostics.Metrics; +using System.Diagnostics; +using OpenTelemetry; +using OpenTelemetry.Metrics; +using System.Collections.Concurrent; + +namespace Azure.Monitor.OpenTelemetry.LiveMetrics.Internals +{ + internal partial class Manager + { + private Meter? _meter; + private MeterProvider? _meterProvider; + private BaseExportingMetricReader? _metricReader; + private readonly ConcurrentQueue> _queue = new(); + + private PerformanceCounter _performanceCounter_ProcessorTime = new PerformanceCounter(categoryName: "Processor", counterName: "% Processor Time", instanceName: "_Total"); + private PerformanceCounter _performanceCounter_CommittedBytes = new PerformanceCounter(categoryName: "Memory", counterName: "Committed Bytes"); + + private Instrument? _myObservableGauge1; + private Instrument? _myObservableGauge2; + + private void InitializeMetrics() + { + var uniqueTestId = Guid.NewGuid(); + + //var meterName = $"meterName{uniqueTestId}"; + var meterName = LiveMetricConstants.LiveMetricMeterName; + _meter = new Meter(meterName, "1.0"); + + _myObservableGauge1 = _meter.CreateObservableGauge(LiveMetricConstants.MemoryCommittedBytesInstrumentName, () => + { + return new Measurement(value: _performanceCounter_CommittedBytes.NextValue()); + }); + + _myObservableGauge2 = _meter.CreateObservableGauge(LiveMetricConstants.ProcessorTimeInstrumentName, () => + { + return new Measurement(value: _performanceCounter_ProcessorTime.NextValue()); + }); + + _metricReader = new BaseExportingMetricReader(new LiveMetricsMetricExporter(_queue)); + + var meterProviderBuilder = Sdk.CreateMeterProviderBuilder() + .AddMeter(meterName) + .AddReader(_metricReader); + + _meterProvider = meterProviderBuilder.Build(); + } + + private IEnumerable CollectMetrics() + { + _metricReader?.Collect(); + + if (_queue.TryDequeue(out var metricPoint)) + { + return metricPoint; + } + else + { + return Array.Empty(); + } + } + } +} diff --git a/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/Internals/Manager.cs b/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/Internals/Manager.cs new file mode 100644 index 0000000000000..610649d878f3b --- /dev/null +++ b/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/Internals/Manager.cs @@ -0,0 +1,207 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Diagnostics; +using System.Threading; +using Azure.Core.Pipeline; +using Azure.Monitor.OpenTelemetry.Exporter.Internals.ConnectionString; +using Azure.Monitor.OpenTelemetry.Exporter.Internals.Platform; +using Azure.Monitor.OpenTelemetry.LiveMetrics.Internals.Diagnostics; +using Azure.Monitor.OpenTelemetry.LiveMetrics.Models; + +namespace Azure.Monitor.OpenTelemetry.LiveMetrics.Internals +{ + internal partial class Manager + { + private readonly QuickPulseSDKClientAPIsRestClient _quickPulseSDKClientAPIsRestClient; + private readonly ConnectionVars _connectionVars; + private readonly bool _isAadEnabled; + private readonly string _streamId = Guid.NewGuid().ToString(); + private Timer _timer; + private string _etag = string.Empty; + private Action _callbackAction = obj => { }; + + public Manager(LiveMetricsExporterOptions options, IPlatform platform) + { + options.Retry.MaxRetries = 0; // prevent Azure.Core from automatically retrying. + + _connectionVars = InitializeConnectionVars(options, platform); + _quickPulseSDKClientAPIsRestClient = InitializeRestClient(options, _connectionVars, out _isAadEnabled); + + _timer = new Timer(callback: OnCallback, state: null, dueTime: Timeout.Infinite, period: Timeout.Infinite); + + if (options.EnableLiveMetrics) + { + SetPingTimer(); + } + + InitializeMetrics(); + } + + internal static ConnectionVars InitializeConnectionVars(LiveMetricsExporterOptions options, IPlatform platform) + { + if (options.ConnectionString == null) + { + var connectionString = platform.GetEnvironmentVariable(EnvironmentVariableConstants.APPLICATIONINSIGHTS_CONNECTION_STRING); + + if (!string.IsNullOrWhiteSpace(connectionString)) + { + return ConnectionStringParser.GetValues(connectionString!); + } + } + else + { + return ConnectionStringParser.GetValues(options.ConnectionString); + } + + throw new InvalidOperationException("A connection string was not found. Please set your connection string."); + } + + private static QuickPulseSDKClientAPIsRestClient InitializeRestClient(LiveMetricsExporterOptions options, ConnectionVars connectionVars, out bool isAadEnabled) + { + HttpPipeline pipeline; + + if (options.Credential != null) + { + var scope = AadHelper.GetScope(connectionVars.AadAudience); + var httpPipelinePolicy = new HttpPipelinePolicy[] + { + new BearerTokenAuthenticationPolicy(options.Credential, scope), + }; + + isAadEnabled = true; + pipeline = HttpPipelineBuilder.Build(options, httpPipelinePolicy); + LiveMetricsExporterEventSource.Log.SetAADCredentialsToPipeline(options.Credential.GetType().Name, scope); + } + else + { + isAadEnabled = false; + pipeline = HttpPipelineBuilder.Build(options); + } + + return new QuickPulseSDKClientAPIsRestClient(new ClientDiagnostics(options), pipeline, host: connectionVars.LiveEndpoint); + } + + private void SetPingTimer() + { + _callbackAction = OnPing; + _timer.Change(dueTime: 0, period: 5000); + } + + private void SetPostTimer() + { + _callbackAction = OnPost; + _timer.Change(dueTime: 0, period: 1000); + } + + private void OnCallback(object state) => _callbackAction.Invoke(state); + + private void OnPing(object state) + { + try + { + Debug.WriteLine($"{DateTime.Now}: OnPing invoked."); + + var response = _quickPulseSDKClientAPIsRestClient.PingCustom( + ikey: _connectionVars.InstrumentationKey, + apikey: null, + xMsQpsTransmissionTime: null, + xMsQpsMachineName: "Desktop-Name", + xMsQpsInstanceName: "Desktop-Name", + xMsQpsStreamId: _streamId, + xMsQpsRoleName: null, + xMsQpsInvariantVersion: "5", + xMsQpsConfigurationEtag: _etag, + monitoringDataPoint: null, + cancellationToken: default); + + if (response.GetRawResponse().Headers.TryGetValue("x-ms-qps-configuration-etag", out string? etagValue) && etagValue != _etag) + { + Debug.WriteLine($"OnPing: updated etag: {etagValue}"); + _etag = etagValue; + } + + if (response.GetRawResponse().Headers.TryGetValue("x-ms-qps-subscribed", out string? subscribedValue) && Convert.ToBoolean(subscribedValue)) + { + Debug.WriteLine($"OnPing: Subscribed: {subscribedValue}"); + SetPostTimer(); + } + } + catch (Exception ex) + { + Debug.WriteLine(ex); + } + } + + /// + /// Send data to LiveMetrics service. + /// + /// + /// + /// dataPoint.Metrics.Add(new MetricPoint { Name = "\\ApplicationInsights\\Requests/Sec", Value = 0, Weight = 1 }); + /// dataPoint.Metrics.Add(new MetricPoint { Name = "\\ApplicationInsights\\Request Duration", Value = 0, Weight = 0 }); + /// dataPoint.Metrics.Add(new MetricPoint { Name = "\\ApplicationInsights\\Requests Failed/Sec", Value = 0, Weight = 1 }); + /// dataPoint.Metrics.Add(new MetricPoint { Name = "\\ApplicationInsights\\Requests Succeeded/Sec", Value = 0, Weight = 1 }); + /// dataPoint.Metrics.Add(new MetricPoint { Name = "\\ApplicationInsights\\Dependency Calls/Sec", Value = 0, Weight = 1 }); + /// dataPoint.Metrics.Add(new MetricPoint { Name = "\\ApplicationInsights\\Dependency Call Duration", Value = 0, Weight = 0 }); + /// dataPoint.Metrics.Add(new MetricPoint { Name = "\\ApplicationInsights\\Dependency Calls Failed/Sec", Value = 0, Weight = 1 }); + /// dataPoint.Metrics.Add(new MetricPoint { Name = "\\ApplicationInsights\\Dependency Calls Succeeded/Sec", Value = 0, Weight = 1 }); + /// dataPoint.Metrics.Add(new MetricPoint { Name = "\\ApplicationInsights\\Exceptions/Sec", Value = 0, Weight = 1 }); + /// dataPoint.Metrics.Add(new MetricPoint { Name = "\\Memory\\Committed Bytes", Value = 41372430336, Weight = 1 }); + /// dataPoint.Metrics.Add(new MetricPoint { Name = "\\Processor(_Total)\\% Processor Time", Value = 14.1891f, Weight = 1 });. + /// + private void OnPost(object state) + { + try + { + Debug.WriteLine($"{DateTime.Now}: OnPost invoked."); + + var dataPoint = new MonitoringDataPoint + { + Version = "DotNetLiveMetricsPOC", // sdk version + MachineName = "Desktop-Name", + Instance = "Desktop-Name", + InvariantVersion = 5, + Timestamp = DateTime.UtcNow, + PerformanceCollectionSupported = false, + IsWebApp = false, + RoleName = null, + StreamId = _streamId, + TransmissionTime = DateTime.UtcNow, + }; + + var metricPoints = CollectMetrics(); + foreach (var metricPoint in metricPoints) + { + dataPoint.Metrics.Add(metricPoint); + } + + var response = _quickPulseSDKClientAPIsRestClient.PostCustom( + ikey: _connectionVars.InstrumentationKey, + apikey: null, + xMsQpsConfigurationEtag: _etag, + xMsQpsTransmissionTime: null, + monitoringDataPoints: new MonitoringDataPoint[] { dataPoint }, + cancellationToken: default); + + if (response.GetRawResponse().Headers.TryGetValue("x-ms-qps-configuration-etag", out string? etagValue) && etagValue != _etag) + { + Debug.WriteLine($"OnPost: updated etag: {etagValue}"); + _etag = etagValue; + } + + if (response.GetRawResponse().Headers.TryGetValue("x-ms-qps-subscribed", out string? subscribedValue) && !Convert.ToBoolean(subscribedValue)) + { + Debug.WriteLine($"OnPost: Subscribed: {subscribedValue}"); + _etag = string.Empty; + SetPingTimer(); + } + } + catch (Exception ex) + { + Debug.WriteLine(ex); + } + } + } +} diff --git a/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/Internals/ManagerFactory.cs b/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/Internals/ManagerFactory.cs new file mode 100644 index 0000000000000..78e630c913e20 --- /dev/null +++ b/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/Internals/ManagerFactory.cs @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System.Collections.Generic; +using Azure.Monitor.OpenTelemetry.Exporter.Internals.Platform; + +namespace Azure.Monitor.OpenTelemetry.LiveMetrics.Internals +{ + internal sealed class ManagerFactory + { + public static readonly ManagerFactory Instance = new(); + + internal readonly Dictionary _runners = new(); + private readonly object _lockObj = new(); + + public Manager Get(LiveMetricsExporterOptions options) + { + var key = options.ConnectionString ?? string.Empty; + + if (!_runners.TryGetValue(key, out Manager? runner)) + { + lock (_lockObj) + { + if (!_runners.TryGetValue(key, out runner)) + { + runner = new Manager(options, new DefaultPlatform()); + + _runners.Add(key, runner); + } + } + } + + return runner; + } + } +} diff --git a/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/LiveMetricConstants.cs b/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/LiveMetricConstants.cs index afa30518c4e1c..99bf2c76918f6 100644 --- a/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/LiveMetricConstants.cs +++ b/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/LiveMetricConstants.cs @@ -1,20 +1,30 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +using System.Collections.Generic; + namespace Azure.Monitor.OpenTelemetry.LiveMetrics { internal static class LiveMetricConstants { internal const string LiveMetricMeterName = "LiveMetricMeterName"; + + // REQUESTS internal const string RequestDurationInstrumentName = "RequestDurationLiveMetric"; internal const string RequestsInstrumentName = "RequestsLiveMetric"; internal const string RequestsSucceededPerSecondInstrumentName = "RequestsSucceededPerSecondLiveMetric"; internal const string RequestsFailedPerSecondInstrumentName = "RequestsFailedPerSecondLiveMetric"; + + // DEPENDENCIES internal const string DependencyDurationInstrumentName = "DependencyDurationLiveMetric"; internal const string DependencyInstrumentName = "DependencyLiveMetric"; internal const string DependencySucceededPerSecondInstrumentName = "DependencySucceededPerSecondLiveMetric"; internal const string DependencyFailedPerSecondInstrumentName = "DependencyFailedPerSecondLiveMetric"; + + // EXCEPTIONS internal const string ExceptionsPerSecondInstrumentName = "ExceptionsPerSecondLiveMetric"; + + // PERFORMANCE COUNTERS internal const string MemoryCommittedBytesInstrumentName = "CommittedBytesLiveMetric"; internal const string ProcessorTimeInstrumentName = "ProcessorTimeBytesLiveMetric"; @@ -29,5 +39,31 @@ internal static class LiveMetricConstants internal const string ExceptionsPerSecondMetricIdValue = @"\ApplicationInsights\Exceptions/Sec"; internal const string MemoryCommittedBytesMetricIdValue = @"\Memory\Committed Bytes"; internal const string ProcessorTimeMetricIdValue = @"\Processor(_Total)\% Processor Time"; + + /// + /// This dictionary maps Instrumentation-Safe names (key) + /// to Application Insights Live Metrics names (value). + /// + internal static readonly Dictionary Mappings = new() + { + // REQUESTS + {RequestsInstrumentName, "\\ApplicationInsights\\Requests/Sec" }, + {RequestDurationInstrumentName, "\\ApplicationInsights\\Request Duration" }, + {RequestsFailedPerSecondInstrumentName, "\\ApplicationInsights\\Requests Failed/Sec" }, + {RequestsSucceededPerSecondInstrumentName, "\\ApplicationInsights\\Requests Succeeded/Sec" }, + + // DEPENDENCIES + {DependencyInstrumentName, "\\ApplicationInsights\\Dependency Calls/Sec" }, + {DependencyDurationInstrumentName, "\\ApplicationInsights\\Dependency Call Duration" }, + {DependencyFailedPerSecondInstrumentName, "\\ApplicationInsights\\Dependency Calls Failed/Sec" }, + {DependencySucceededPerSecondInstrumentName, "\\ApplicationInsights\\Dependency Calls Succeeded/Sec" }, + + // EXCEPTIONS + {ExceptionsPerSecondInstrumentName, "\\ApplicationInsights\\Exceptions/Sec" }, + + // PERFORMANCE COUNTERS + {MemoryCommittedBytesInstrumentName, "\\Memory\\Committed Bytes"}, + {ProcessorTimeInstrumentName, "\\Processor(_Total)\\% Processor Time"}, + }; } } diff --git a/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/LiveMetricsExporterOptions.cs b/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/LiveMetricsExporterOptions.cs index 576d5481b2830..111990b0bf969 100644 --- a/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/LiveMetricsExporterOptions.cs +++ b/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/LiveMetricsExporterOptions.cs @@ -10,7 +10,7 @@ namespace Azure.Monitor.OpenTelemetry.LiveMetrics; /// /// Options that allow users to configure the Live Metrics. /// -internal class LiveMetricsExporterOptions : ClientOptions +public class LiveMetricsExporterOptions : ClientOptions { /// /// The Connection String provides users with a single configuration setting to identify the Azure Monitor resource and endpoint. @@ -23,5 +23,10 @@ internal class LiveMetricsExporterOptions : ClientOptions /// /// Enables or disables the Live Metrics feature. /// - public bool EnableLiveMetrics { get; set; } + public bool EnableLiveMetrics { get; set; } = true; + + /// + /// Get or sets the value of . + /// + public TokenCredential Credential { get; set; } } diff --git a/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/LiveMetricsExtensions.cs b/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/LiveMetricsExtensions.cs index 1129423957e43..0f9c36fe45058 100644 --- a/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/LiveMetricsExtensions.cs +++ b/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/LiveMetricsExtensions.cs @@ -14,7 +14,7 @@ namespace Azure.Monitor.OpenTelemetry.LiveMetrics /// /// Extension methods to register Live Metrics. /// - internal static class LiveMetricsExtensions + public static class LiveMetricsExtensions { /// /// Adds Live Metrics to the TracerProvider. diff --git a/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/LiveMetricsExtractionProcessor.cs b/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/LiveMetricsExtractionProcessor.cs index 79d26541138a6..4153b85ba0a4f 100644 --- a/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/LiveMetricsExtractionProcessor.cs +++ b/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/LiveMetricsExtractionProcessor.cs @@ -20,6 +20,7 @@ internal sealed class LiveMetricsExtractionProcessor : BaseProcessor private LiveMetricsResource? _resource; internal readonly MeterProvider? _meterProvider; private readonly Meter _meter; + private readonly Counter _requests; private readonly Histogram _requestDuration; private readonly Counter _requestSucceededPerSecond; @@ -29,6 +30,7 @@ internal sealed class LiveMetricsExtractionProcessor : BaseProcessor private readonly Counter _dependencySucceededPerSecond; private readonly Counter _dependencyFailedPerSecond; private readonly Counter _exceptionsPerSecond; + private readonly DoubleBuffer _doubleBuffer; internal static readonly IReadOnlyDictionary s_liveMetricNameMapping = new Dictionary() @@ -50,23 +52,29 @@ internal sealed class LiveMetricsExtractionProcessor : BaseProcessor internal LiveMetricsExtractionProcessor(DoubleBuffer doubleBuffer, LiveMetricsExporter liveMetricExporter) { - _meterProvider = Sdk.CreateMeterProviderBuilder() - .AddMeter(LiveMetricConstants.LiveMetricMeterName) - .AddReader(new PeriodicExportingMetricReader(exporter: liveMetricExporter, exportIntervalMilliseconds:5000) - { TemporalityPreference = MetricReaderTemporalityPreference.Delta }) - // TODO: Remove Console Exporter - .AddConsoleExporter() - .Build(); + //_meterProvider = Sdk.CreateMeterProviderBuilder() + // .AddMeter(LiveMetricConstants.LiveMetricMeterName) + // .AddReader(new PeriodicExportingMetricReader(exporter: liveMetricExporter, exportIntervalMilliseconds:5000) + // { TemporalityPreference = MetricReaderTemporalityPreference.Delta }) + // // TODO: Remove Console Exporter + // .AddConsoleExporter() + // .Build(); _meter = new Meter(LiveMetricConstants.LiveMetricMeterName); + + // REQUEST _requests = _meter.CreateCounter(LiveMetricConstants.RequestsInstrumentName); _requestDuration = _meter.CreateHistogram(LiveMetricConstants.RequestDurationInstrumentName); _requestSucceededPerSecond = _meter.CreateCounter(LiveMetricConstants.RequestsSucceededPerSecondInstrumentName); _requestFailedPerSecond = _meter.CreateCounter(LiveMetricConstants.RequestsFailedPerSecondInstrumentName); + + // DEPENDENCY _dependency = _meter.CreateCounter(LiveMetricConstants.DependencyInstrumentName); _dependencyDuration = _meter.CreateHistogram(LiveMetricConstants.DependencyDurationInstrumentName); _dependencySucceededPerSecond = _meter.CreateCounter(LiveMetricConstants.DependencySucceededPerSecondInstrumentName); _dependencyFailedPerSecond = _meter.CreateCounter(LiveMetricConstants.DependencyFailedPerSecondInstrumentName); + + // EXCEPTIONS _exceptionsPerSecond = _meter.CreateCounter(LiveMetricConstants.ExceptionsPerSecondInstrumentName); _doubleBuffer = doubleBuffer; } @@ -98,11 +106,11 @@ public override void OnEnd(Activity activity) _requestDuration.Record(activity.Duration.TotalMilliseconds); if (IsSuccess(activity, statusCodeAttributeValue)) { - _requestFailedPerSecond.Add(1); + _requestSucceededPerSecond.Add(1); } else { - _requestSucceededPerSecond.Add(1); + _requestFailedPerSecond.Add(1); } AddRequestDocument(activity, statusCodeAttributeValue); @@ -117,11 +125,11 @@ public override void OnEnd(Activity activity) _dependencyDuration.Record(activity.Duration.TotalMilliseconds); if (IsSuccess(activity, statusCodeAttributeValue)) { - _dependencyFailedPerSecond.Add(1); + _dependencySucceededPerSecond.Add(1); } else { - _dependencySucceededPerSecond.Add(1); + _dependencyFailedPerSecond.Add(1); } AddRemoteDependencyDocument(activity); diff --git a/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/LiveMetricsMetricExporter.cs b/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/LiveMetricsMetricExporter.cs new file mode 100644 index 0000000000000..a3293e3dc6ceb --- /dev/null +++ b/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/LiveMetricsMetricExporter.cs @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System.Collections; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using Azure.Monitor.OpenTelemetry.LiveMetrics.Internals; +using OpenTelemetry; +using OpenTelemetry.Metrics; + +namespace Azure.Monitor.OpenTelemetry.LiveMetrics +{ + /// + /// Converts OTel Metrics to Azure Monitor Metrics. + /// + internal class LiveMetricsMetricExporter : BaseExporter + { + private readonly ConcurrentQueue> _metricPoints; + + /// + /// TODO. + /// + public LiveMetricsMetricExporter(ConcurrentQueue> metricPoints) + { + _metricPoints = metricPoints; + } + + /// + public override ExportResult Export(in Batch batch) + { + var list = new List(capacity: (int)batch.Count); // TODO: POSSIBLE OVERFLOW EXCEPTION (long -> int) + + //Debug.Assert(batch.Count == 11); + foreach (var metric in batch) + { + foreach (ref readonly var metricPoint in metric.GetMetricPoints()) + { + //Debug.WriteLine(LiveMetricConstants.Mappings[metric.Name]); + + switch (metric.MetricType) + { + case MetricType.LongSum: + list.Add(new Models.MetricPoint + { + Name = LiveMetricConstants.Mappings[metric.Name], + Value = metricPoint.GetSumLong(), + Weight = 1 + }); + break; + case MetricType.Histogram: + list.Add(new Models.MetricPoint + { + Name = LiveMetricConstants.Mappings[metric.Name], + Value = (float)metricPoint.GetHistogramSum(), + Weight = (int)metricPoint.GetHistogramCount() // TODO: POSSIBLE OVERFLOW EXCEPTION (long -> int) + }); + break; + case MetricType.DoubleGauge: + list.Add(new Models.MetricPoint + { + Name = LiveMetricConstants.Mappings[metric.Name], + Value = (float)metricPoint.GetGaugeLastValueDouble(), + Weight = 1 + }); + break; + default: + Debug.WriteLine($"Unsupported Metric Type {metric.MetricType} {metric.Name}"); + break; + } + } + } + //Debug.Assert(list.Count == 11); + + _metricPoints.Enqueue(list); + Debug.Write($"Enqueue {_metricPoints.Count}. Count {list.Count}\n"); + + return ExportResult.Success; + } + } +} diff --git a/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/LiveMetricsTraceExporter.cs b/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/LiveMetricsTraceExporter.cs new file mode 100644 index 0000000000000..de6c315d6b016 --- /dev/null +++ b/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/src/LiveMetricsTraceExporter.cs @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System.Diagnostics; +using Azure.Monitor.OpenTelemetry.LiveMetrics.Internals; +using OpenTelemetry; + +namespace Azure.Monitor.OpenTelemetry.LiveMetrics +{ + /// + /// Empty Exporter. This is used to enable the Manager Singleton. + /// + public class LiveMetricsTraceExporter : BaseExporter + { + private readonly Manager _manager; + + /// + /// TODO. + /// + public LiveMetricsTraceExporter(LiveMetricsExporterOptions options) : this(options, ManagerFactory.Instance.Get(options)) + { + } + + internal LiveMetricsTraceExporter(LiveMetricsExporterOptions options, Manager manager) + { + _manager = manager; + } + + /// + public override ExportResult Export(in Batch batch) + { + return ExportResult.Success; + } + } +} diff --git a/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/tests/Azure.Monitor.OpenTelemetry.LiveMetrics.Demo/Program.cs b/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/tests/Azure.Monitor.OpenTelemetry.LiveMetrics.Demo/Program.cs index b8ac8ac7d3cf7..ebdc7f84e7769 100644 --- a/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/tests/Azure.Monitor.OpenTelemetry.LiveMetrics.Demo/Program.cs +++ b/sdk/monitor/Azure.Monitor.OpenTelemetry.LiveMetrics/tests/Azure.Monitor.OpenTelemetry.LiveMetrics.Demo/Program.cs @@ -2,14 +2,92 @@ // Licensed under the MIT License. using System; +using OpenTelemetry.Trace; +using OpenTelemetry; +using System.Diagnostics; namespace Azure.Monitor.OpenTelemetry.LiveMetrics.Demo { internal class Program { + private const string ActivitySourceName = "MyCompany.MyProduct.MyLibrary"; + private static readonly ActivitySource s_activitySource = new(ActivitySourceName); + + private const string ConnectionString = "InstrumentationKey=00000000-0000-0000-0000-000000000000"; + + private static Random _random = new(); + public static void Main(string[] args) { - Console.WriteLine("Hello, World!"); + var options = new LiveMetricsExporterOptions { ConnectionString = ConnectionString }; + + using TracerProvider tracerProvider = Sdk.CreateTracerProviderBuilder() + .AddSource(ActivitySourceName) + .AddProcessor(new SimpleActivityExportProcessor(new LiveMetricsTraceExporter(options: options))) + .AddLiveMetrics(configure => configure.ConnectionString = ConnectionString) + .Build(); + + Console.WriteLine("Press any key to stop the loop."); + + // Loop until a key is pressed + while (!Console.KeyAvailable) + { + GenerateTelemetry(); + System.Threading.Thread.Sleep(200); + } + + Console.WriteLine("Key pressed. Exiting the loop."); + } + + private static bool GetRandomBool(int percent) => percent >= _random.Next(0, 100); + + private static void GenerateTelemetry() + { + // Request + if (GetRandomBool(percent: 70)) + { + Console.WriteLine("Request"); + using (var activity = s_activitySource.StartActivity("Request", kind: ActivityKind.Server)) + { + // Exception + if (GetRandomBool(percent: 40)) + { + Console.WriteLine("Request Exception"); + try + { + throw new Exception("Test exception"); + } + catch (Exception ex) + { + activity?.SetStatus(ActivityStatusCode.Error); + activity?.RecordException(ex); + } + } + } + } + + // Dependency + if (GetRandomBool(percent: 70)) + { + Console.WriteLine("Dependency"); + using (var activity = s_activitySource.StartActivity("Dependency", kind: ActivityKind.Client)) + { + // Exception + if (GetRandomBool(percent: 40)) + { + Console.WriteLine("Dependency Exception"); + try + { + throw new Exception("Test exception"); + } + catch (Exception ex) + { + activity?.SetStatus(ActivityStatusCode.Error); + activity?.RecordException(ex); + } + } + } + } } } }