[Dotnet Monitor] Ignore - Adding System.Diagnostics.Metrics Support (#…

…3529)

* Got basic counter rate end-to-end working - currently in a broken state as I investigate other types of metrics

* Leftovers from previous commit

* Gauges working for systems diagnostics metrics

* Added in histogram, adding in options for maxHistograms and maxTimeSeries

* Added in error payloads for logging purposes

* Temporarily changed visibility for testing - this may be reverted later

* Now handling multiple sessions

* Added filtering for counters, instead of allowing all counters for a provider to go through.

* Handle observable... errors

* Some cleanup, added error event check

* Update dependencies from https://github.com/dotnet/source-build-reference-packages build 20221129.1 (#3528)

[main] Update dependencies from dotnet/source-build-reference-packages

Co-authored-by: dotnet-maestro[bot] <42748379+dotnet-maestro[bot]@users.noreply.github.com>

eng/Version.Details.xml

@@ -12,9 +12,9 @@
       <Uri>https://github.com/microsoft/clrmd</Uri>
       <Sha>a64d9ac11086f28fbd4b2b2337c19be7826fbfa9</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.SourceBuild.Intermediate.source-build-reference-packages" Version="8.0.0-alpha.1.22578.1">
+    <Dependency Name="Microsoft.SourceBuild.Intermediate.source-build-reference-packages" Version="8.0.0-alpha.1.22579.1">
       <Uri>https://github.com/dotnet/source-build-reference-packages</Uri>
-      <Sha>7ec4b0aabc55efa11515c987fb0251d39c29f06c</Sha>
+      <Sha>bed0fd20a5a55843c1f04ef195ef2fd7171ae715</Sha>
       <SourceBuild RepoName="source-build-reference-packages" ManagedOnly="true" />
     </Dependency>
   </ProductDependencies>

src/Microsoft.Diagnostics.Monitoring.EventPipe/Configuration/MetricSourceConfiguration.cs

@@ -16,6 +16,7 @@ namespace Microsoft.Diagnostics.Monitoring.EventPipe
     public sealed class MetricSourceConfiguration : MonitoringSourceConfiguration
     {
         private readonly IList<EventPipeProvider> _eventPipeProviders;
+        public string SessionId { get; private set; }
 
         public MetricSourceConfiguration(float metricIntervalSeconds, IEnumerable<string> customProviderNames)
         {
@@ -45,6 +46,37 @@ public MetricSourceConfiguration(float metricIntervalSeconds, IEnumerable<string
                })).ToList();
         }
 
+        public MetricSourceConfiguration(float metricIntervalSeconds, IEnumerable<string> customProviderNames, int maxHistograms, int maxTimeSeries) : this(metricIntervalSeconds, customProviderNames)
+        {
+            const long TimeSeriesValues = 0x2;
+            StringBuilder metrics = new StringBuilder();
+            foreach (string provider in customProviderNames)
+            {
+                if (metrics.Length != 0)
+                {
+                    metrics.Append(",");
+                }
+
+                metrics.Append(provider);
+            }
+
+            SessionId = Guid.NewGuid().ToString();
+
+            EventPipeProvider metricsEventSourceProvider =
+                new EventPipeProvider("System.Diagnostics.Metrics", EventLevel.Informational, TimeSeriesValues,
+                    new Dictionary<string, string>()
+                    {
+                        { "SessionId", SessionId },
+                        { "Metrics", metrics.ToString() },
+                        { "RefreshInterval", MetricIntervalSeconds.ToString() },
+                        { "MaxTimeSeries", maxTimeSeries.ToString() },
+                        { "MaxHistograms", maxHistograms.ToString() }
+                    }
+                );
+
+            _eventPipeProviders = _eventPipeProviders.Append(metricsEventSourceProvider).ToArray();
+        }
+
         private string MetricIntervalSeconds { get; }
 
         public override IList<EventPipeProvider> GetProviders() => _eventPipeProviders;

src/Microsoft.Diagnostics.Monitoring.EventPipe/Counters/CounterFilter.cs

@@ -44,6 +44,12 @@ public bool IsIncluded(string providerName, string counterName, int intervalMill
             {
                 return false;
             }
+
+            return IsIncluded(providerName, counterName);
+        }
+
+        public bool IsIncluded(string providerName, string counterName)
+        {
             if (_enabledCounters.Count == 0)
             {
                 return true;

src/Microsoft.Diagnostics.Monitoring.EventPipe/Counters/CounterPayload.cs

@@ -8,7 +8,7 @@
 
 namespace Microsoft.Diagnostics.Monitoring.EventPipe
 {
-    internal class CounterPayload : ICounterPayload
+    public class CounterPayload : ICounterPayload
     {
 #if NETSTANDARD
         private static readonly IReadOnlyDictionary<string, string> Empty = new ReadOnlyDictionary<string, string>(new Dictionary<string, string>(0));
@@ -35,13 +35,26 @@ public CounterPayload(DateTime timestamp,
             Provider = provider;
             Interval = interval;
             Metadata = metadata ?? Empty;
+            EventType = EventType.Gauge;
+        }
+
+        // Copied from dotnet-counters
+        public CounterPayload(string providerName, string name, string displayName, string displayUnits, Dictionary<string, string> metadata, double value, DateTime timestamp, string type, EventType eventType)
+        {
+            Provider = providerName;
+            Name = name;
+            Metadata = metadata ?? Empty;
+            Value = value;
+            Timestamp = timestamp;
+            CounterType = (CounterType)Enum.Parse(typeof(CounterType), type);
+            EventType = eventType;
         }
 
         public string Namespace { get; }
 
         public string Name { get; }
 
-        public string DisplayName { get; }
+        public string DisplayName { get; protected set; }
 
         public string Unit { get; }
 
@@ -56,5 +69,63 @@ public CounterPayload(DateTime timestamp,
         public string Provider { get; }
 
         public IReadOnlyDictionary<string, string> Metadata { get; }
+
+        public EventType EventType { get; set; }
+
+    }
+
+    public class GaugePayload : CounterPayload
+    {
+        public GaugePayload(string providerName, string name, string displayName, string displayUnits, Dictionary<string, string> metadata, double value, DateTime timestamp) :
+            base(providerName, name, displayName, displayUnits, metadata, value, timestamp, "Metric", EventType.Gauge)
+        {
+            // In case these properties are not provided, set them to appropriate values.
+            string counterName = string.IsNullOrEmpty(displayName) ? name : displayName;
+            DisplayName = !string.IsNullOrEmpty(displayUnits) ? $"{counterName} ({displayUnits})" : counterName;
+        }
+    }
+
+    public class RatePayload : CounterPayload
+    {
+        public RatePayload(string providerName, string name, string displayName, string displayUnits, Dictionary<string, string> metadata, double value, double intervalSecs, DateTime timestamp) :
+            base(providerName, name, displayName, displayUnits, metadata, value, timestamp, "Rate", EventType.Rate)
+        {
+            // In case these properties are not provided, set them to appropriate values.
+            string counterName = string.IsNullOrEmpty(displayName) ? name : displayName;
+            string unitsName = string.IsNullOrEmpty(displayUnits) ? "Count" : displayUnits;
+            string intervalName = intervalSecs.ToString() + " sec";
+            DisplayName = $"{counterName} ({unitsName} / {intervalName})";
+        }
+    }
+
+    public class PercentilePayload : CounterPayload
+    {
+        public PercentilePayload(string providerName, string name, string displayName, string displayUnits, Dictionary<string, string> metadata, double val, DateTime timestamp) :
+            base(providerName, name, displayName, displayUnits, metadata, val, timestamp, "Metric", EventType.Histogram)
+        {
+            // In case these properties are not provided, set them to appropriate values.
+            string counterName = string.IsNullOrEmpty(displayName) ? name : displayName;
+            DisplayName = !string.IsNullOrEmpty(displayUnits) ? $"{counterName} ({displayUnits})" : counterName;
+        }
+    }
+
+    public class ErrorPayload : CounterPayload
+    {
+        public ErrorPayload(string providerName, string name, string displayName, string displayUnits, Dictionary<string, string> metadata, double val, DateTime timestamp, string errorMessage) :
+            base(providerName, name, displayName, displayUnits, metadata, val, timestamp, "Metric", EventType.Error)
+        {
+            ErrorMessage = errorMessage;
+        }
+
+        public string ErrorMessage { get; private set; }
+    }
+
+    // If keep this, should probably put it somewhere else
+    public enum EventType : int
+    {
+        Rate,
+        Gauge,
+        Histogram,
+        Error
     }
 }

src/Microsoft.Diagnostics.Monitoring.EventPipe/Counters/EventCounterPipeline.cs

@@ -15,6 +15,7 @@ internal class EventCounterPipeline : EventSourcePipeline<EventPipeCounterPipeli
     {
         private readonly IEnumerable<ICountersLogger> _loggers;
         private readonly CounterFilter _filter;
+        private string _sessionId;
 
         public EventCounterPipeline(DiagnosticsClient client,
             EventPipeCounterPipelineSettings settings,
@@ -38,7 +39,11 @@ public EventCounterPipeline(DiagnosticsClient client,
 
         protected override MonitoringSourceConfiguration CreateConfiguration()
         {
-            return new MetricSourceConfiguration(Settings.CounterIntervalSeconds, _filter.GetProviders());
+            var config = new MetricSourceConfiguration(Settings.CounterIntervalSeconds, _filter.GetProviders(), Settings.MaxHistograms, Settings.MaxTimeSeries);
+
+            _sessionId = config.SessionId;
+
+            return config;
         }
 
         protected override async Task OnEventSourceAvailable(EventPipeEventSource eventSource, Func<Task> stopSessionAsync, CancellationToken token)
@@ -49,7 +54,7 @@ protected override async Task OnEventSourceAvailable(EventPipeEventSource eventS
             {
                 try
                 {
-                    if (traceEvent.TryGetCounterPayload(_filter, out ICounterPayload counterPayload))
+                    if (traceEvent.TryGetCounterPayload(_filter, _sessionId, out List<ICounterPayload> counterPayload))
                     {
                         ExecuteCounterLoggerAction((metricLogger) => metricLogger.Log(counterPayload));
                     }

src/Microsoft.Diagnostics.Monitoring.EventPipe/Counters/EventPipeCounterPipelineSettings.cs

@@ -15,6 +15,10 @@ internal class EventPipeCounterPipelineSettings : EventSourcePipelineSettings
         //Do not use TimeSpan here since we may need to synchronize this pipeline interval
         //with a different session and want to make sure the values are identical.
         public float CounterIntervalSeconds { get; set; }
+
+        public int MaxHistograms { get; set; }
+
+        public int MaxTimeSeries { get; set; }
     }
 
     internal class EventPipeCounterGroup

src/Microsoft.Diagnostics.Monitoring.EventPipe/Counters/ICounterPayload.cs

@@ -8,7 +8,7 @@
 
 namespace Microsoft.Diagnostics.Monitoring.EventPipe
 {
-    internal enum CounterType
+    public enum CounterType
     {
         //Same as average or mean
         Metric,
@@ -17,7 +17,7 @@ internal enum CounterType
         Rate
     }
 
-    internal interface ICounterPayload
+    public interface ICounterPayload
     {
         string Name { get; }
 
@@ -36,5 +36,7 @@ internal interface ICounterPayload
         float Interval { get; }
 
         IReadOnlyDictionary<string, string> Metadata { get; }
+
+        public EventType EventType { get; set; }
     }
 }

src/Microsoft.Diagnostics.Monitoring.EventPipe/Counters/ICountersLogger.cs

@@ -14,7 +14,7 @@ internal interface ICountersLogger
     {
         //TODO Consider making these async.
 
-        void Log(ICounterPayload counter);
+        void Log(List<ICounterPayload> counter);
         void PipelineStarted();
         void PipelineStopped();
     }