Allow per provider interval specification (dotnet#3591)

* Allow per provider interval specification

* Update src/Microsoft.Diagnostics.Monitoring.WebApi/Metrics/MetricsSettingsFactory.cs

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

* Update src/Tests/Microsoft.Diagnostics.Monitoring.Tool.UnitTests/MetricsSettingsTests.cs

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

* Apply suggestions from code review

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

* Pr feedback

* pr feedback

* pr feedback

---------

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

documentation/configuration/metrics-configuration.md

@@ -8,7 +8,7 @@
 Due to limitations in event counters, `dotnet monitor` supports only **one** refresh interval when collecting metrics. This interval is used for
 Prometheus metrics, livemetrics, triggers, traces, and trigger actions that collect traces. The default interval is 5 seconds, but can be changed in configuration.
 
-[8.0+] For EventCounter providers, is possible to specify a different interval for each provider. See [Per provider intervals](#per-provider-intervals-71).
+[7.1+] For EventCounter providers, is possible to specify a different interval for each provider. See [Per provider intervals](#per-provider-intervals-71).
 
 <details>
   <summary>JSON</summary>
@@ -39,7 +39,7 @@ Prometheus metrics, livemetrics, triggers, traces, and trigger actions that coll
   ```
 </details>
 
-## Per provider intervals (8.0+)
+## Per provider intervals (7.1+)
 
 It is possible to override the global interval on a per provider basis. Note this forces all scenarios (triggers, live metrics, prometheus metrics, traces) that use a particular provider to use that interval. Metrics that are `System.Diagnostics.Metrics` based always use global interval.
 

documentation/schema.json

@@ -962,6 +962,31 @@
           "default": 1000,
           "maximum": 2147483647.0,
           "minimum": 1.0
+        },
+        "Providers": {
+          "type": [
+            "null",
+            "object"
+          ],
+          "description": "Dictionary of provider names and their global configuration.",
+          "additionalProperties": {
+            "$ref": "#/definitions/GlobalProviderOptions"
+          }
+        }
+      }
+    },
+    "GlobalProviderOptions": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "IntervalSeconds": {
+          "type": [
+            "null",
+            "number"
+          ],
+          "format": "float",
+          "maximum": 86400.0,
+          "minimum": 1.0
         }
       }
     },

src/Microsoft.Diagnostics.Monitoring.Options/GlobalCounterOptions.cs

@@ -2,12 +2,15 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using System;
+using System.Collections.Generic;
 using System.ComponentModel;
 using System.ComponentModel.DataAnnotations;
+using System.Globalization;
+using System.Linq;
 
 namespace Microsoft.Diagnostics.Monitoring.WebApi
 {
-    public class GlobalCounterOptions
+    public partial class GlobalCounterOptions
     {
         public const float IntervalMinSeconds = 1;
         public const float IntervalMaxSeconds = 60 * 60 * 24; // One day
@@ -32,6 +35,38 @@ public class GlobalCounterOptions
         [DefaultValue(GlobalCounterOptionsDefaults.MaxTimeSeries)]
         [Range(1, int.MaxValue)]
         public int? MaxTimeSeries { get; set; }
+
+        [Display(
+            ResourceType = typeof(OptionsDisplayStrings),
+            Description = nameof(OptionsDisplayStrings.DisplayAttributeDescription_GlobalCounterOptions_Providers))]
+        public System.Collections.Generic.IDictionary<string, GlobalProviderOptions> Providers { get; set; } = new Dictionary<string, GlobalProviderOptions>(StringComparer.OrdinalIgnoreCase);
+    }
+
+    public class GlobalProviderOptions
+    {
+        [Range(GlobalCounterOptions.IntervalMinSeconds, GlobalCounterOptions.IntervalMaxSeconds)]
+        public float? IntervalSeconds { get; set; }
+    }
+
+    partial class GlobalCounterOptions : IValidatableObject
+    {
+        public IEnumerable<ValidationResult> Validate(ValidationContext validationContext)
+        {
+            var results = new List<ValidationResult>();
+            var providerResults = new List<ValidationResult>();
+            foreach ((string provider, GlobalProviderOptions options) in Providers)
+            {
+                providerResults.Clear();
+                if (!Validator.TryValidateObject(options, new ValidationContext(options), providerResults, true))
+                {
+                    // We prefix the validation error with the provider.
+                    results.AddRange(providerResults.Select(r => new ValidationResult(
+                        string.Format(CultureInfo.CurrentCulture, OptionsDisplayStrings.ErrorMessage_NestedProviderValidationError, provider, r.ErrorMessage))));
+                }
+            }
+
+            return results;
+        }
     }
 
     internal static class GlobalCounterOptionsExtensions
@@ -44,5 +79,8 @@ public static int GetMaxHistograms(this GlobalCounterOptions options) =>
 
         public static int GetMaxTimeSeries(this GlobalCounterOptions options) =>
             options.MaxTimeSeries.GetValueOrDefault(GlobalCounterOptionsDefaults.MaxTimeSeries);
+
+        public static float GetProviderSpecificInterval(this GlobalCounterOptions options, string providerName) =>
+            options.Providers.TryGetValue(providerName, out GlobalProviderOptions providerOptions) ? providerOptions.IntervalSeconds ?? options.GetIntervalSeconds() : options.GetIntervalSeconds();
     }
 }

src/Microsoft.Diagnostics.Monitoring.Options/OptionsDisplayStrings.resx

@@ -804,4 +804,7 @@
   <data name="DisplayAttributeDescription_MetricsOptions_Meters" xml:space="preserve">
     <value>Names of meters to collect from the System.Diagnostics.Metrics provider.</value>
   </data>
-</root>
+  <data name="DisplayAttributeDescription_GlobalCounterOptions_Providers" xml:space="preserve">
+    <value>Dictionary of provider names and their global configuration.</value>
+  </data>
+</root>

src/Microsoft.Diagnostics.Monitoring.WebApi/Controllers/DiagController.cs

@@ -337,7 +337,7 @@ public Task<ActionResult> CaptureTrace(
             {
                 TimeSpan duration = Utilities.ConvertSecondsToTimeSpan(durationSeconds);
 
-                var aggregateConfiguration = TraceUtilities.GetTraceConfiguration(profile, _counterOptions.CurrentValue.GetIntervalSeconds());
+                var aggregateConfiguration = TraceUtilities.GetTraceConfiguration(profile, _counterOptions.CurrentValue);
 
                 return StartTrace(processInfo, aggregateConfiguration, duration, egressProvider, tags);
             }, processKey, Utilities.ArtifactType_Trace);

src/Microsoft.Diagnostics.Monitoring.WebApi/Metrics/MetricsSettingsFactory.cs

@@ -4,6 +4,7 @@
 using Microsoft.Diagnostics.Monitoring.EventPipe;
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Linq;
 using System.Threading;
 
@@ -20,6 +21,7 @@ public static MetricsPipelineSettings CreateSettings(GlobalCounterOptions counte
             return CreateSettings(includeDefaults,
                 durationSeconds,
                 counterOptions.GetIntervalSeconds(),
+                counterOptions.Providers,
                 counterOptions.GetMaxHistograms(),
                 counterOptions.GetMaxTimeSeries(),
                 () => new List<EventPipeCounterGroup>(0));
@@ -29,6 +31,7 @@ public static MetricsPipelineSettings CreateSettings(GlobalCounterOptions counte
         {
             return CreateSettings(options.IncludeDefaultProviders.GetValueOrDefault(MetricsOptionsDefaults.IncludeDefaultProviders),
                 Timeout.Infinite, counterOptions.GetIntervalSeconds(),
+                counterOptions.Providers,
                 counterOptions.GetMaxHistograms(),
                 counterOptions.GetMaxTimeSeries(),
                 () => ConvertCounterGroups(options.Providers, options.Meters));
@@ -40,6 +43,7 @@ public static MetricsPipelineSettings CreateSettings(GlobalCounterOptions counte
             return CreateSettings(configuration.IncludeDefaultProviders,
                 durationSeconds,
                 counterOptions.GetIntervalSeconds(),
+                counterOptions.Providers,
                 counterOptions.GetMaxHistograms(),
                 counterOptions.GetMaxTimeSeries(),
                 () => ConvertCounterGroups(configuration.Providers, configuration.Meters));
@@ -48,6 +52,7 @@ public static MetricsPipelineSettings CreateSettings(GlobalCounterOptions counte
         private static MetricsPipelineSettings CreateSettings(bool includeDefaults,
             int durationSeconds,
             float counterInterval,
+            IDictionary<string, GlobalProviderOptions> intervalMap,
             int maxHistograms,
             int maxTimeSeries,
             Func<List<EventPipeCounterGroup>> createCounterGroups)
@@ -61,6 +66,15 @@ private static MetricsPipelineSettings CreateSettings(bool includeDefaults,
                 eventPipeCounterGroups.Add(new EventPipeCounterGroup { ProviderName = MonitoringSourceConfiguration.GrpcAspNetCoreServer, Type = CounterGroupType.EventCounter });
             }
 
+            foreach (EventPipeCounterGroup counterGroup in eventPipeCounterGroups)
+            {
+                if (intervalMap.TryGetValue(counterGroup.ProviderName, out GlobalProviderOptions providerInterval))
+                {
+                    Debug.Assert(counterGroup.IntervalSeconds == null, "Unexpected value for provider interval");
+                    counterGroup.IntervalSeconds = providerInterval.IntervalSeconds;
+                }
+            }
+
             return new MetricsPipelineSettings
             {
                 CounterGroups = eventPipeCounterGroups.ToArray(),

src/Microsoft.Diagnostics.Monitoring.WebApi/Strings.resx

@@ -136,7 +136,7 @@
     <comment>Gets a string similar to "Invalid metric count.".</comment>
   </data>
   <data name="ErrorMessage_InvalidMetricInterval" xml:space="preserve">
-    <value>Custom trace metric provider '{0}' must use the global counter interval '{1}'</value>
+    <value>Custom trace metric provider '{0}' must use the expected counter interval '{1}'.</value>
   </data>
   <data name="ErrorMessage_MetricsDisabled" xml:space="preserve">
     <value>Metrics was not enabled.</value>

src/Microsoft.Diagnostics.Monitoring.WebApi/Utilities/TraceUtilities.cs

@@ -13,7 +13,7 @@ namespace Microsoft.Diagnostics.Monitoring.WebApi
 {
     internal static class TraceUtilities
     {
-        public static MonitoringSourceConfiguration GetTraceConfiguration(Models.TraceProfile profile, float metricsIntervalSeconds)
+        public static MonitoringSourceConfiguration GetTraceConfiguration(Models.TraceProfile profile, GlobalCounterOptions options)
         {
             var configurations = new List<MonitoringSourceConfiguration>();
             if (profile.HasFlag(Models.TraceProfile.Cpu))
@@ -34,7 +34,14 @@ public static MonitoringSourceConfiguration GetTraceConfiguration(Models.TracePr
             }
             if (profile.HasFlag(Models.TraceProfile.Metrics))
             {
-                configurations.Add(new MetricSourceConfiguration(metricsIntervalSeconds, Enumerable.Empty<string>()));
+                IEnumerable<MetricEventPipeProvider> defaultProviders = MonitoringSourceConfiguration.DefaultMetricProviders.Select(provider => new MetricEventPipeProvider
+                {
+                    Provider = provider,
+                    IntervalSeconds = options.GetProviderSpecificInterval(provider),
+                    Type = MetricType.EventCounter
+                });
+
+                configurations.Add(new MetricSourceConfiguration(options.GetIntervalSeconds(), defaultProviders));
             }
 
             return new AggregateSourceConfiguration(configurations.ToArray());

src/Microsoft.Diagnostics.Monitoring.WebApi/Validation/CounterValidator.cs

@@ -17,12 +17,12 @@ public static bool ValidateProvider(GlobalCounterOptions counterOptions,
             if (provider.Arguments?.TryGetValue("EventCounterIntervalSec", out string intervalValue) == true)
             {
                 if (float.TryParse(intervalValue, out float intervalSeconds) &&
-                    intervalSeconds != counterOptions.GetIntervalSeconds())
+                    intervalSeconds != counterOptions.GetProviderSpecificInterval(provider.Name))
                 {
                     errorMessage = string.Format(CultureInfo.CurrentCulture,
                         Strings.ErrorMessage_InvalidMetricInterval,
                         provider.Name,
-                        counterOptions.GetIntervalSeconds());
+                        counterOptions.GetProviderSpecificInterval(provider.Name));
                     return false;
                 }
             }

src/Tests/Microsoft.Diagnostics.Monitoring.Tool.FunctionalTests/Options/OptionsExtensions.cs

@@ -45,6 +45,15 @@ public static RootOptions AddGlobalCounter(this RootOptions options, int interva
             return options;
         }
 
+        public static RootOptions AddProviderInterval(this RootOptions options, string name, int intervalSeconds)
+        {
+            Assert.NotNull(options.GlobalCounter);
+
+            options.GlobalCounter.Providers.Add(name, new GlobalProviderOptions { IntervalSeconds = (float)intervalSeconds });
+
+            return options;
+        }
+
         public static CollectionRuleOptions CreateCollectionRule(this RootOptions rootOptions, string name)
         {
             CollectionRuleOptions options = new();

src/Tests/Microsoft.Diagnostics.Monitoring.Tool.UnitTests/CollectionRuleOptionsTests.cs

@@ -1,6 +1,7 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using Microsoft.Diagnostics.Monitoring.EventPipe;
 using Microsoft.Diagnostics.Monitoring.TestCommon;
 using Microsoft.Diagnostics.Monitoring.TestCommon.Options;
 using Microsoft.Diagnostics.Monitoring.WebApi.Models;
@@ -19,6 +20,7 @@
 using System.Diagnostics.Tracing;
 using System.Globalization;
 using System.Linq;
+using System.Reflection;
 using System.Threading.Tasks;
 using Xunit;
 using Xunit.Abstractions;
@@ -1055,6 +1057,65 @@ public Task CollectionRuleOptions_CollectTraceAction_PropertyValidation()
                 });
         }
 
+        [Fact]
+        public Task CollectionRuleOptions_CollectTraceAction_ValidateProviderIntervals()
+        {
+            const string ExpectedEgressProvider = "TmpEgressProvider";
+            const int ExpectedInterval = 7;
+
+            return ValidateFailure(
+                rootOptions =>
+                {
+                    rootOptions.AddGlobalCounter(5);
+                    rootOptions.AddProviderInterval(MonitoringSourceConfiguration.SystemRuntimeEventSourceName, ExpectedInterval);
+
+                    rootOptions.AddFileSystemEgress(ExpectedEgressProvider, "/tmp");
+
+                    rootOptions.CreateCollectionRule(DefaultRuleName)
+                        .SetStartupTrigger()
+                        .AddCollectTraceAction(new EventPipeProvider[] { new EventPipeProvider
+                        {
+                            Name = MonitoringSourceConfiguration.SystemRuntimeEventSourceName,
+                            Arguments = new Dictionary<string, string>{ { "EventCounterIntervalSec", "5" } },
+                        }},
+                        ExpectedEgressProvider, null);
+                },
+                ex =>
+                {
+                    string failure = Assert.Single(ex.Failures);
+                    VerifyProviderIntervalMessage(failure, MonitoringSourceConfiguration.SystemRuntimeEventSourceName, ExpectedInterval);
+                });
+        }
+
+        [Fact]
+        public Task CollectionRuleOptions_CollectTraceAction_InvalidProviderInterval()
+        {
+            const string ExpectedEgressProvider = "TmpEgressProvider";
+
+            return ValidateFailure(
+                rootOptions =>
+                {
+                    rootOptions.AddGlobalCounter(5);
+                    rootOptions.AddProviderInterval(MonitoringSourceConfiguration.SystemRuntimeEventSourceName, -2);
+
+                    rootOptions.AddFileSystemEgress(ExpectedEgressProvider, "/tmp");
+
+                    rootOptions.CreateCollectionRule(DefaultRuleName)
+                        .SetStartupTrigger()
+                        .AddCollectTraceAction(new EventPipeProvider[] { new EventPipeProvider
+                        {
+                            Name = MonitoringSourceConfiguration.SystemRuntimeEventSourceName,
+                            Arguments = new Dictionary<string, string>{ { "EventCounterIntervalSec", "5" } },
+                        }},
+                        ExpectedEgressProvider, null);
+                },
+                ex =>
+                {
+                    string failure = Assert.Single(ex.Failures);
+                    VerifyNestedGlobalInterval(failure, MonitoringSourceConfiguration.SystemRuntimeEventSourceName);
+                });
+        }
+
         [Fact]
         public Task CollectionRuleOptions_CollectTraceAction_NoProfileOrProviders()
         {
@@ -1884,5 +1945,23 @@ private static void VerifyMissingStoppingEventProviderMessage(string[] failures,
 
             Assert.Equal(message, failures[index]);
         }
+
+        private static void VerifyProviderIntervalMessage(string failure, string provider, int expectedInterval)
+        {
+            string message = string.Format(CultureInfo.CurrentCulture, WebApi.Strings.ErrorMessage_InvalidMetricInterval, provider, expectedInterval);
+
+            Assert.Equal(message, failure);
+        }
+
+        private static void VerifyNestedGlobalInterval(string failure, string provider)
+        {
+            string rangeValidationMessage = typeof(WebApi.GlobalProviderOptions)
+                .GetProperty(nameof(WebApi.GlobalProviderOptions.IntervalSeconds))
+                .GetCustomAttribute<RangeAttribute>()
+                .FormatErrorMessage(nameof(WebApi.GlobalProviderOptions.IntervalSeconds));
+
+            string message = string.Format(CultureInfo.CurrentCulture, WebApi.OptionsDisplayStrings.ErrorMessage_NestedProviderValidationError, provider, rangeValidationMessage);
+            Assert.Equal(message, failure);
+        }
     }
 }