Allow for Metric Scraping interval to be defined overall & per-metric via YAML config

adding-a-new-scraper.md

@@ -10,6 +10,7 @@ This guide walks you through the process of adding a new scraper type.
 ## Configuration
 1. Add your new scraping type to the  `Promitor.Core.Scraping.Configuration.Model.ResourceType`
 2. Describe the resource for which you're creating scraping metrics by creating `<New-Type>MetricDefinition`and inherit from `Promitor.Core.Scraping.Configuration.Model.Metrics.MetricDefinition` - this class should go in `.\src\Promitor.Core.Scraping\Configuration\Model\Metrics\ResourceTypes`
+2. Create an example entry for your scraper in `.\samples\promitor-sample.yaml`
 3. Create a new Deserializer in `.\src\Promitor.Core.Scraping\Configuration\Serialization\Deserializers`. This must inherit from `AzureMetricDeserializer`.
 3. Update `Promitor.Core.Scraping.Factories.MetricDeserializerFactory` to handle your new resource type by returning a new instance of the Deserializer you created in the previous step.
 4. Provide a unit test in `.\src\Promitor.Scraper.Tests.Unit\Serialization\MetricsDeclaration\` that tests the deserialization based on our sample

docs/configuration/index.md

@@ -1,4 +1,3 @@
----
 layout: default
 title: General Configuration
 ---
@@ -15,7 +14,6 @@ Promitor automatically scrapes Azure Monitor and makes the information available
 The behavior of this can be configured with the following environment variables:
 - **PROMITOR_CONFIGURATION_PATH** - Defines the location of the YAML file that declares what Azure Monitor metrics to scrape. If nothing is specified, `/config/metrics-declaration.yaml` will be used.
 - **PROMITOR_SCRAPE_BASEPATH** - Controls the path where the scraping endpoint for Prometheus is being exposed. If nothing is specified, `/prometheus/scrape` will be used.
-- **PROMITOR_SCRAPE_SCHEDULE** - A cron expression that controls the fequency in which all the configured metrics will be scraped from Azure Monitor. You can use [crontab-generator.org](https://crontab-generator.org/) to generate a cron that fits your needs. If nothing is configured, `*/5 * * * *` will be used.
 
 We're also providing feature flags to opt-out of certain features:
 - **PROMITOR_FEATURE_METRICSTIMESTAMP** - Defines whether or not a timestamp should be included when the value was scraped on Azure Monitor. Supported values are `True` to opt-in & `False` to opt-out, if nothing is configured this will be turned on.

docs/configuration/metrics/index.md

@@ -16,12 +16,18 @@ azureMetadata:
 metricDefaults:
   aggregation:
     interval: 00:05:00
+  scraping:
+    # Every minute
+    schedule: "0 * * ? * *"
 metrics:
   - name: demo_queue_size
     description: "Amount of active messages of the 'myqueue' queue"
     resourceType: ServiceBusQueue
     namespace: promitor-messaging
     queueName: orders
+    scraping:
+      # Every 2 minutes
+      schedule: "0 */2 * ? * *"
     azureMetricConfiguration:
       metricName: ActiveMessages
       aggregation:
@@ -50,6 +56,7 @@ metrics:
 
 ## Metric Defaults
 
+- `metricDefaults.scraping.schedule` - **[REQUIRED]** A cron expression that controls the fequency in which all the configured metrics will be scraped from Azure Monitor. You can use [crontab-generator.org](https://crontab-generator.org/) to generate a cron that fits your needs.
 - `metricDefaults.aggregation.interval` - The default interval which defines over what period measurements of a metric should be aggregated.
 
 ## Metrics
@@ -64,6 +71,10 @@ Every metric that is being declared needs to define the following fields:
 - `azureMetricConfiguration.aggregation.type` - The aggregation that needs to be used when querying Azure Monitor
 - `azureMetricConfiguration.aggregation.interval` - Overrides the default aggregation interval defined in `metricDefaults.aggregation.interval` with a new interval
 
+Additionally, the following fields are optional:
+
+- `scraping.schedule` - A scraping schedule for the individual metric; overrides the the one specified in `metricDefaults`
+
 # Supported Azure Services
 
 Every Azure service is supported and can be scraped by using the [Generic Azure Resource](generic-azure-resource).

samples/promitor-sample.yaml

@@ -5,6 +5,9 @@ azureMetadata:
 metricDefaults:
   aggregation:
     interval: 00:05:00
+  scraping:
+    # Every minute
+    schedule: "0 * * ? * *"
 metrics:
   - name: demo_generic_queue_size
     description: "Amount of active messages of the 'myqueue' queue (determined with Generic provider)"
@@ -20,6 +23,9 @@ metrics:
     resourceType: Generic
     resourceUri: Microsoft.ServiceBus/namespaces/promitor-messaging
     # filter is deliberately omitted given filter is optional
+    scraping:
+      # Every 2 minutes
+      schedule: "0 */2 * ? * *"
     azureMetricConfiguration:
       metricName: ActiveMessages
       aggregation:
@@ -41,6 +47,9 @@ metrics:
     accountName: promitor
     queueName: orders
     sasToken: "?sv=2018-03-28&ss=bfqt&srt=sco&sp=rwdlacup&se=2019-07-28T02:33:14Z&st=2019-03-24T18:33:14Z&spr=https&sig=OiwNEYueCWlOhveapM1K6cRgV%2Be21gNhoq%2FDZqJEMZE%3D"
+    scraping:
+      # Every 2 minutes
+      schedule: "0 */2 * ? * *"
     azureMetricConfiguration:
       metricName: MessageCount
       aggregation:

src/Promitor.Core.Scraping/Configuration/Model/Scraping.cs

@@ -1,9 +1,7 @@
-using System;
-
-namespace Promitor.Core.Scraping.Configuration.Model
+namespace Promitor.Core.Scraping.Configuration.Model
 {
     public class Scraping
     {
-        public TimeSpan? Interval { get; set; }
+        public string Schedule { get; set; }
     }
 }

src/Promitor.Core.Scraping/Configuration/Providers/Interfaces/IMetricsDeclarationProvider.cs

@@ -7,7 +7,9 @@ public interface IMetricsDeclarationProvider
         /// <summary>
         ///     Gets the configured metrics declaration
         /// </summary>
-        MetricsDeclaration Get();
+        /// <param name="applyDefaults"><c>true</c> if the provider should apply default values from top-level 
+        /// configuration elements to metrics where those values aren't specified. <c>false</c> otherwise</param>
+        MetricsDeclaration Get(bool applyDefaults = false);
 
         /// <summary>
         ///     Gets the serialized metrics declaration

src/Promitor.Core.Scraping/Configuration/Providers/MetricsDeclarationProvider.cs

@@ -16,11 +16,49 @@ public MetricsDeclarationProvider(ILogger logger)
             _configurationSerializer = new ConfigurationSerializer(logger);
         }
 
-        public virtual MetricsDeclaration Get()
+        public virtual MetricsDeclaration Get(bool applyDefaults = false)
         {
             var rawMetricsDeclaration = ReadRawDeclaration();
 
             var config = _configurationSerializer.Deserialize(rawMetricsDeclaration);
+
+            if (applyDefaults)
+            {
+                foreach (var metric in config.Metrics)
+                {
+                    // Apply AzureMetadata.ResourceGroupName to metrics with no other RG specified
+                    if (string.IsNullOrWhiteSpace(metric.ResourceGroupName))
+                    {
+                        metric.ResourceGroupName = config.AzureMetadata.ResourceGroupName;
+                    }
+
+                    // Apply the default aggregation interval if none is specified
+                    if (metric.AzureMetricConfiguration == null)
+                    {
+                        metric.AzureMetricConfiguration = new AzureMetricConfiguration();
+                    }
+                    if (metric.AzureMetricConfiguration?.Aggregation == null)
+                    {
+                        metric.AzureMetricConfiguration.Aggregation = new MetricAggregation();
+                    }
+                    if (metric.AzureMetricConfiguration?.Aggregation.Interval == null)
+                    {
+                        metric.AzureMetricConfiguration.Aggregation.Interval = config.MetricDefaults.Aggregation.Interval;
+                    }
+
+                    // Apply the default scraping interval if none is specified
+                    if (metric.Scraping == null)
+                    {
+                        metric.Scraping = config.MetricDefaults.Scraping;
+                    }
+
+                    // Apply the default scraping interval if none is specified
+                    if (metric.Scraping.Schedule == null)
+                    {
+                        metric.Scraping.Schedule = config.MetricDefaults.Scraping.Schedule;
+                    }
+                }
+            }
             return config;
         }
 

src/Promitor.Core.Scraping/Configuration/Serialization/Core/MetricDefaultsDeserializer.cs

@@ -1,6 +1,4 @@
-using System;
-using System.Collections.Generic;
-using GuardNet;
+using GuardNet;
 using Microsoft.Extensions.Logging;
 using Promitor.Core.Scraping.Configuration.Model;
 using YamlDotNet.RepresentationModel;
@@ -29,22 +27,10 @@ internal override MetricDefaults Deserialize(YamlMappingNode node)
 
             if (node.Children.ContainsKey(@"scraping"))
             {
-                var scrapingNode = (YamlMappingNode)node.Children[new YamlScalarNode(@"scraping")];
-                try
-                {
-                    var scrapingIntervalNode = scrapingNode.Children[new YamlScalarNode(@"interval")];
-
-                    if (scrapingIntervalNode != null)
-                    {
-                        var scrapingIntervalTimeSpan = TimeSpan.Parse(scrapingIntervalNode.ToString());
-                        metricDefaults.Scraping.Interval = scrapingIntervalTimeSpan;
-                    }
-                }
-                catch (KeyNotFoundException)
-                {
-                    // happens when the YAML doesn't have the properties in it which is fine because the object
-                    // will get a default interval of 'null'
-                }
+                var metricDefaultsNode = (YamlMappingNode)node.Children[new YamlScalarNode("scraping")];
+                var metricDefaultsSerializer = new ScrapingDeserializer(Logger);
+                var scraping = metricDefaultsSerializer.Deserialize(metricDefaultsNode);
+                metricDefaults.Scraping = scraping;
             }
 
             return metricDefaults;

src/Promitor.Core.Scraping/Configuration/Serialization/Core/MetricDeserializer.cs

@@ -1,5 +1,4 @@
-using System;
-using System.Collections.Generic;
+using System.Collections.Generic;
 using GuardNet;
 using Microsoft.Extensions.Logging;
 using Promitor.Core.Scraping.Configuration.Model.Metrics;
@@ -45,11 +44,11 @@ protected virtual TMetricDefinition DeserializeMetricDefinition<TMetricDefinitio
                 var scrapingNode = (YamlMappingNode)metricNode.Children[new YamlScalarNode(@"scraping")];
                 try
                 {
-                    var scrapingIntervalNode = scrapingNode?.Children[new YamlScalarNode(@"interval")];
+                    var scrapingIntervalNode = scrapingNode?.Children[new YamlScalarNode(@"schedule")];
 
                     if (scrapingIntervalNode != null)
                     {
-                        metricDefinition.Scraping.Interval = TimeSpan.Parse(scrapingIntervalNode.ToString());
+                        metricDefinition.Scraping.Schedule = scrapingIntervalNode.ToString();
                     }
                 }
                 catch (KeyNotFoundException)

src/Promitor.Core.Scraping/Configuration/Serialization/Core/ScrapingDeserializer.cs

@@ -0,0 +1,29 @@
+using Microsoft.Extensions.Logging;
+using YamlDotNet.RepresentationModel;
+
+namespace Promitor.Core.Scraping.Configuration.Serialization.Core
+{
+    internal class ScrapingDeserializer : Deserializer<Model.Scraping>
+    {
+        internal ScrapingDeserializer(ILogger logger) : base(logger)
+        {
+        }
+
+        internal override Model.Scraping Deserialize(YamlMappingNode node)
+        {
+            var scraping = new Model.Scraping();
+
+            if (node.Children.ContainsKey("schedule"))
+            {
+                var rawScheduleNode = node.Children[new YamlScalarNode("schedule")];
+                scraping.Schedule = rawScheduleNode.ToString();
+            }
+            else
+            {
+                Logger.LogError("No default metric scraping schedule was configured!");
+            }
+
+            return scraping;
+        }
+    }
+}

src/Promitor.Core.Scraping/Factories/MetricScraperFactory.cs

@@ -16,31 +16,30 @@ public class MetricScraperFactory
         /// </summary>
         /// <param name="azureMetadata">Metadata concerning the Azure resources</param>
         /// <param name="metricDefinitionResourceType">Resource type to scrape</param>
-        /// <param name="metricDefaults">Default configuration for metrics</param>
         /// <param name="logger">General logger</param>
         /// <param name="exceptionTracker">Tracker used to log exceptions</param>
         public static IScraper<MetricDefinition> CreateScraper(ResourceType metricDefinitionResourceType, AzureMetadata azureMetadata,
-            MetricDefaults metricDefaults, ILogger logger, IExceptionTracker exceptionTracker)
+            ILogger logger, IExceptionTracker exceptionTracker)
         {
             var azureCredentials = DetermineAzureCredentials();
             var azureMonitorClient = CreateAzureMonitorClient(azureMetadata, azureCredentials, logger);
 
             switch (metricDefinitionResourceType)
             {
                 case ResourceType.ServiceBusQueue:
-                    return new ServiceBusQueueScraper(azureMetadata, metricDefaults, azureMonitorClient, logger, exceptionTracker);
+                    return new ServiceBusQueueScraper(azureMetadata, azureMonitorClient, logger, exceptionTracker);
                 case ResourceType.Generic:
-                    return new GenericScraper(azureMetadata, metricDefaults, azureMonitorClient, logger, exceptionTracker);
+                    return new GenericScraper(azureMetadata, azureMonitorClient, logger, exceptionTracker);
                 case ResourceType.StorageQueue:
-                    return new StorageQueueScraper(azureMetadata, metricDefaults, azureMonitorClient, logger, exceptionTracker);
+                    return new StorageQueueScraper(azureMetadata, azureMonitorClient, logger, exceptionTracker);
                 case ResourceType.ContainerInstance:
-                    return new ContainerInstanceScraper(azureMetadata, metricDefaults, azureMonitorClient, logger, exceptionTracker);
+                    return new ContainerInstanceScraper(azureMetadata, azureMonitorClient, logger, exceptionTracker);
                 case ResourceType.VirtualMachine:
-                    return new VirtualMachineScraper(azureMetadata, metricDefaults, azureMonitorClient, logger, exceptionTracker);
+                    return new VirtualMachineScraper(azureMetadata, azureMonitorClient, logger, exceptionTracker);
                 case ResourceType.NetworkInterface:
-                    return new NetworkInterfaceScraper(azureMetadata, metricDefaults, azureMonitorClient, logger, exceptionTracker);
+                    return new NetworkInterfaceScraper(azureMetadata, azureMonitorClient, logger, exceptionTracker);
                 case ResourceType.ContainerRegistry:
-                    return new ContainerRegistryScraper(azureMetadata, metricDefaults, azureMonitorClient, logger, exceptionTracker);
+                    return new ContainerRegistryScraper(azureMetadata, azureMonitorClient, logger, exceptionTracker);
                 default:
                     throw new ArgumentOutOfRangeException();
             }

src/Promitor.Core.Scraping/ResourceTypes/ContainerInstanceScraper.cs

@@ -13,8 +13,8 @@ public class ContainerInstanceScraper : Scraper<ContainerInstanceMetricDefinitio
     {
         private const string ResourceUriTemplate = "subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.ContainerInstance/containerGroups/{2}";
 
-        public ContainerInstanceScraper(AzureMetadata azureMetadata, MetricDefaults metricDefaults, AzureMonitorClient azureMonitorClient, ILogger logger, IExceptionTracker exceptionTracker)
-            : base(azureMetadata, metricDefaults, azureMonitorClient, logger, exceptionTracker)
+        public ContainerInstanceScraper(AzureMetadata azureMetadata, AzureMonitorClient azureMonitorClient, ILogger logger, IExceptionTracker exceptionTracker)
+            : base(azureMetadata, azureMonitorClient, logger, exceptionTracker)
         {
         }
 

src/Promitor.Core.Scraping/ResourceTypes/ContainerRegistryScraper.cs

@@ -13,8 +13,8 @@ public class ContainerRegistryScraper : Scraper<ContainerRegistryMetricDefinitio
     {
         private const string ResourceUriTemplate = "subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.ContainerRegistry/registries/{2}";
 
-        public ContainerRegistryScraper(AzureMetadata azureMetadata, MetricDefaults metricDefaults, AzureMonitorClient azureMonitorClient, ILogger logger, IExceptionTracker exceptionTracker)
-            : base(azureMetadata, metricDefaults, azureMonitorClient, logger, exceptionTracker)
+        public ContainerRegistryScraper(AzureMetadata azureMetadata, AzureMonitorClient azureMonitorClient, ILogger logger, IExceptionTracker exceptionTracker)
+            : base(azureMetadata, azureMonitorClient, logger, exceptionTracker)
         {
         }
 

src/Promitor.Core.Scraping/ResourceTypes/GenericScraper.cs

@@ -13,8 +13,8 @@ internal class GenericScraper : Scraper<GenericAzureMetricDefinition>
     {
         private const string ResourceUriTemplate = "subscriptions/{0}/resourceGroups/{1}/providers/{2}";
 
-        public GenericScraper(AzureMetadata azureMetadata, MetricDefaults metricDefaults, AzureMonitorClient azureMonitorClient, ILogger logger, IExceptionTracker exceptionTracker)
-            : base(azureMetadata, metricDefaults, azureMonitorClient, logger, exceptionTracker)
+        public GenericScraper(AzureMetadata azureMetadata, AzureMonitorClient azureMonitorClient, ILogger logger, IExceptionTracker exceptionTracker)
+            : base(azureMetadata, azureMonitorClient, logger, exceptionTracker)
         {
         }
 

src/Promitor.Core.Scraping/ResourceTypes/NetworkInterfaceScraper.cs

@@ -13,8 +13,8 @@ internal class NetworkInterfaceScraper : Scraper<NetworkInterfaceMetricDefinitio
     {
         private const string ResourceUriTemplate = "subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.Network/networkInterfaces/{2}";
 
-        public NetworkInterfaceScraper(AzureMetadata azureMetadata, MetricDefaults metricDefaults, AzureMonitorClient azureMonitorClient, ILogger logger, IExceptionTracker exceptionTracker)
-            : base(azureMetadata, metricDefaults, azureMonitorClient, logger, exceptionTracker)
+        public NetworkInterfaceScraper(AzureMetadata azureMetadata, AzureMonitorClient azureMonitorClient, ILogger logger, IExceptionTracker exceptionTracker)
+            : base(azureMetadata, azureMonitorClient, logger, exceptionTracker)
         {
         }
 

src/Promitor.Core.Scraping/ResourceTypes/ServiceBusQueueScraper.cs

@@ -13,8 +13,8 @@ public class ServiceBusQueueScraper : Scraper<ServiceBusQueueMetricDefinition>
     {
         private const string ResourceUriTemplate = "subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.ServiceBus/namespaces/{2}";
 
-        public ServiceBusQueueScraper(AzureMetadata azureMetadata, MetricDefaults metricDefaults, AzureMonitorClient azureMonitorClient, ILogger logger, IExceptionTracker exceptionTracker)
-            : base(azureMetadata, metricDefaults, azureMonitorClient, logger, exceptionTracker)
+        public ServiceBusQueueScraper(AzureMetadata azureMetadata, AzureMonitorClient azureMonitorClient, ILogger logger, IExceptionTracker exceptionTracker)
+            : base(azureMetadata, azureMonitorClient, logger, exceptionTracker)
         {
         }
 

src/Promitor.Core.Scraping/ResourceTypes/StorageQueueScraper.cs

@@ -14,8 +14,8 @@ namespace Promitor.Core.Scraping.ResourceTypes
     public class StorageQueueScraper : Scraper<StorageQueueMetricDefinition>
     {
         private readonly AzureStorageQueueClient _azureStorageQueueClient;
-        public StorageQueueScraper(AzureMetadata azureMetadata, MetricDefaults metricDefaults, AzureMonitorClient azureMonitorClient, ILogger logger, IExceptionTracker exceptionTracker)
-            : base(azureMetadata, metricDefaults, azureMonitorClient, logger, exceptionTracker)
+        public StorageQueueScraper(AzureMetadata azureMetadata, AzureMonitorClient azureMonitorClient, ILogger logger, IExceptionTracker exceptionTracker)
+            : base(azureMetadata, azureMonitorClient, logger, exceptionTracker)
         {
             _azureStorageQueueClient = new AzureStorageQueueClient(logger);
         }