Improve metric determination for storage queues and streamline metric docs

docs/configuration/metrics/generic-azure-resource.md

@@ -23,7 +23,8 @@ resourceUri: Microsoft.ServiceBus/namespaces/promitor-messaging
 filter: EntityName eq 'orders'
 azureMetricConfiguration:
   metricName: ActiveMessages
-  aggregation: Total
+  aggregation:
+    type: Total
 ```
 
 [&larr; back to metrics declarations](/configuration/metrics)<br />

docs/configuration/metrics/index.md

@@ -13,6 +13,9 @@ azureMetadata:
   tenantId: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
   subscriptionId: yyyyyyyy-yyyy-yyyy-yyyy-yyyyyyyyyyyy
   resourceGroupName: promitor
+metricDefaults:
+  aggregation:
+    interval: 00:05:00
 metrics: 
   - name: demo_queue_size
     description: "Amount of active messages of the 'myqueue' queue"
@@ -21,7 +24,9 @@ metrics:
     queueName: orders
     azureMetricConfiguration:
       metricName: ActiveMessages
-      aggregation: Total
+      aggregation:
+        type: Total
+        interval: 00:15:00
 ```
 
 # General Declaration
@@ -31,20 +36,25 @@ metrics:
 - `azureMetadata.subscriptionId` - The id of the default subscription to query.
 - `azureMetadata.resourceGroupName` - The name of the default resource group to query.
 
+## Metric Defaults
+
+- `metricDefaults.aggregation.interval` - The default interval which defines over what period measurements of a metric should be aggregated.
+
 ## Metrics
 Every metric that is being declared needs to define the following fields:
 - `name` - Name of the metric that will be exposed in the scrape endpoint for Prometheus
 - `description` - Description for the metric that will be exposed in the scrape endpoint for Prometheus
 - `resourceType` - Defines what type of resource needs to be queried.
 - `azureMetricConfiguration.metricName` - The name of the metric in Azure Monitor to query
-- `azureMetricConfiguration.aggregation` - The aggregation that needs to be used when querying Azure Monitor
+- `azureMetricConfiguration.aggregation.type` - The aggregation that needs to be used when querying Azure Monitor
+- `azureMetricConfiguration.aggregation.interval` - Overrides the default aggregation interval defined in `metricDefaults.aggregation.interval` with a new interval
 
 # Supported Azure Services
 Every Azure service is supported and can be scraped by using the [Generic Azure Resource](generic-azure-resource).
 
 We also provide a simplified way to configure the following Azure resources:
 - [Azure Service Bus Queue](service-bus-queue)
-- [Azure Storage Queue](azure-storage-queue)
+- [Azure Storage Queue](storage-queue)
 
 Want to help out? Create an issue and [contribute a new scraper](https://github.com/tomkerkhove/promitor/blob/master/adding-a-new-scraper.md).
 

docs/configuration/metrics/service-bus-queue.md

@@ -11,11 +11,11 @@ The following fields need to be provided:
 - `queueName` - The name of the queue
 
 Supported metrics:
-- IncomingMessages
-- IncomingRequests
-- ActiveMessages
-- Messages
-- Size
+- `IncomingMessages`
+- `IncomingRequests`
+- `ActiveMessages`
+- `Messages`
+- `Size`
 
 Example:
 ```yaml
@@ -26,7 +26,8 @@ namespace: promitor-messaging
 queueName: orders
 azureMetricConfiguration:
   metricName: ActiveMessages
-  aggregation: Total
+  aggregation:
+    type: Total
 ```
 
 [&larr; back to metrics declarations](/configuration/metrics)<br />

docs/configuration/metrics/azure-storage-queue.md → docs/configuration/metrics/storage-queue.md

@@ -12,10 +12,8 @@ The following fields need to be provided:
 - `sasToken` - The SAS token used to access the queue/account
 
 Supported metrics:
-- MessageCount
-- TimeSpentInQueue
-
-TimeSpentInQueue - Time in seconds that the oldest message has been waiting in the queue to be processed.
+- `TimeSpentInQueue` - Time in seconds that the oldest message has been waiting in the queue to be processed.
+- `MessageCount`
 
 Example:
 ```yaml
@@ -26,7 +24,8 @@ accountName: promitor
 queueName: orders
 azureMetricConfiguration:
   metricName: MessageCount
-  aggregation: Total
+  aggregation:
+    type: Total
 ```
 
 [&larr; back to metrics declarations](/configuration/metrics)<br />

samples/promitor-sample.yaml

@@ -36,12 +36,22 @@ metrics:
         # Optionally override the default aggregation interval (metricDefaults.aggregation.interval)
         interval: 00:15:00
   - name: demo_azurestoragequeue_queue_size
-    description: "Approximate amount of messages in 'testqueue' queue (determined with StorageQueue provider)"
+    description: "Approximate amount of messages in 'orders' queue (determined with StorageQueue provider)"
     resourceType: StorageQueue
-    accountName: testaccount
-    queueName: testqueue
-    sasToken: "?sv=2015-04-05&si=read&sig=zs87c3nUp1uiF4UAMMstXrLKhIpGOirFHRcQ4XYxxpY%3D"
+    accountName: promitor
+    queueName: orders
+    sasToken: "?sv=2018-03-28&ss=bfqt&srt=sco&sp=rwdlacup&se=2019-07-28T02:33:14Z&st=2019-03-24T18:33:14Z&spr=https&sig=OiwNEYueCWlOhveapM1K6cRgV%2Be21gNhoq%2FDZqJEMZE%3D"
     azureMetricConfiguration:
       metricName: MessageCount
       aggregation:
         type: Total
+  - name: demo_azurestoragequeue_queue_timespentinqueue
+    description: "Approximate amount of time that the oldest message has been in 'orders' queue (determined with StorageQueue provider)"
+    resourceType: StorageQueue
+    accountName: promitor
+    queueName: orders
+    sasToken: "?sv=2018-03-28&ss=bfqt&srt=sco&sp=rwdlacup&se=2019-07-28T02:33:14Z&st=2019-03-24T18:33:14Z&spr=https&sig=OiwNEYueCWlOhveapM1K6cRgV%2Be21gNhoq%2FDZqJEMZE%3D"
+    azureMetricConfiguration:
+      metricName: TimeSpentInQueue
+      aggregation:
+        type: Total

src/Promitor.Core.Scraping/Constants.cs

@@ -5,8 +5,6 @@ public static class Constants
         public static class Defaults
         {
             public const string MetricsDeclarationPath = "/config/metrics-declaration.yaml";
-            public const string TimeSpentInQueueMetricName = "TimeSpentInQueue";
-            public const string MessageCountMetricName = "MessageCount";
         }
     }
 }

src/Promitor.Core.Scraping/ResourceTypes/InvalidMetricNameException.cs

@@ -4,13 +4,25 @@ namespace Promitor.Core.Scraping.ResourceTypes
 {
     public class InvalidMetricNameException : Exception
     {
-        public string MetricName { get; }
-        public string ResourceProviderName { get; }
-
+        /// <summary>
+        ///     Constructor
+        /// </summary>
+        /// <param name="metricName">Name of the metric that was not valid</param>
+        /// <param name="resourceProviderName">Name of the resource provider that was configured, ie. Service Bus Queues</param>
         public InvalidMetricNameException(string metricName, string resourceProviderName) : base($"Invalid metric name '{metricName}' for {resourceProviderName}")
         {
             MetricName = metricName;
             ResourceProviderName = resourceProviderName;
         }
+
+        /// <summary>
+        ///     Name of the metric that was not valid
+        /// </summary>
+        public string MetricName { get; }
+
+        /// <summary>
+        ///     Name of the resource provider that was configured
+        /// </summary>
+        public string ResourceProviderName { get; }
     }
 }

src/Promitor.Core.Scraping/ResourceTypes/StorageQueueScraper.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Threading.Tasks;
+using GuardNet;
 using Microsoft.Azure.Management.Monitor.Fluent.Models;
 using Microsoft.Extensions.Logging;
 using Promitor.Core.Scraping.Configuration.Model;
@@ -21,11 +22,15 @@ public StorageQueueScraper(AzureMetadata azureMetadata, MetricDefaults metricDef
 
         protected override async Task<double> ScrapeResourceAsync(StorageQueueMetricDefinition metricDefinition, AggregationType aggregationType, TimeSpan aggregationInterval)
         {
-            switch (metricDefinition.AzureMetricConfiguration.MetricName)
+            Guard.NotNull(metricDefinition, nameof(metricDefinition));
+            Guard.NotNull(metricDefinition.AzureMetricConfiguration, nameof(metricDefinition.AzureMetricConfiguration));
+            Guard.NotNullOrEmpty(metricDefinition.AzureMetricConfiguration.MetricName, nameof(metricDefinition.AzureMetricConfiguration.MetricName));
+
+            switch (metricDefinition.AzureMetricConfiguration.MetricName.ToLowerInvariant())
             {
-                case Constants.Defaults.TimeSpentInQueueMetricName:
+                case AzureStorageConstants.Queues.Metrics.TimeSpentInQueue:
                     return await _azureStorageQueueClient.GetQueueMessageTimeSpentInQueueAsync(metricDefinition.AccountName, metricDefinition.QueueName, metricDefinition.SasToken);
-                case Constants.Defaults.MessageCountMetricName:
+                case AzureStorageConstants.Queues.Metrics.MessageCount:
                     return await _azureStorageQueueClient.GetQueueMessageCountAsync(metricDefinition.AccountName, metricDefinition.QueueName, metricDefinition.SasToken);
                 default:
                     throw new InvalidMetricNameException(metricDefinition.AzureMetricConfiguration.MetricName, metricDefinition.ResourceType.ToString());

src/Promitor.Integrations.AzureStorage/AzureStorageConstants.cs

@@ -0,0 +1,14 @@
+namespace Promitor.Integrations.AzureStorage
+{
+    public static class AzureStorageConstants
+    {
+        public static class Queues
+        {
+            public static class Metrics
+            {
+                public const string MessageCount = "messagecount";
+                public const string TimeSpentInQueue = "timespentinqueue";
+            }
+        }
+    }
+}

src/Promitor.Integrations.AzureStorage/AzureStorageQueueClient.cs

@@ -32,7 +32,9 @@ public async Task<int> GetQueueMessageCountAsync(string accountName, string queu
 
             await queue.FetchAttributesAsync();
             var messageCount = queue.ApproximateMessageCount ?? 0;
+
             _logger.LogInformation("Current size of queue {0} is {1}", queueName, messageCount);
+
             return messageCount;
         }
 
@@ -49,6 +51,7 @@ public async Task<double> GetQueueMessageTimeSpentInQueueAsync(string accountNam
 
             var msg = await queue.PeekMessageAsync();
             var timeSpentInQueue = msg.InsertionTime.HasValue ? DateTime.UtcNow - msg.InsertionTime.Value.UtcDateTime : TimeSpan.Zero;
+
             return timeSpentInQueue.TotalSeconds;
         }
 

src/Promitor.Scraper.Host/Promitor.Scraper.Host.csproj

@@ -51,6 +51,7 @@
     <ProjectReference Include="..\Promitor.Core\Promitor.Core.csproj" />
     <ProjectReference Include="..\Promitor.Core.Telemetry\Promitor.Core.Telemetry.csproj" />
     <ProjectReference Include="..\Promitor.Integrations.AzureMonitor\Promitor.Integrations.AzureMonitor.csproj" />
+    <ProjectReference Include="..\Promitor.Integrations.AzureStorage\Promitor.Integrations.AzureStorage.csproj" />
   </ItemGroup>
 
 </Project>

src/Promitor.Scraper.Host/Validation/MetricDefinitions/ResourceTypes/StorageQueueMetricValidator.cs

@@ -1,37 +1,40 @@
+using System;
 using System.Collections.Generic;
+using System.Linq;
 using Promitor.Core.Scraping.Configuration.Model.Metrics.ResourceTypes;
+using Promitor.Integrations.AzureStorage;
 using Promitor.Scraper.Host.Validation.MetricDefinitions.Interfaces;
 
 namespace Promitor.Scraper.Host.Validation.MetricDefinitions.ResourceTypes
 {
-    public class StorageQueueMetricValidator: IMetricValidator<StorageQueueMetricDefinition>
+    public class StorageQueueMetricValidator : IMetricValidator<StorageQueueMetricDefinition>
     {
         private readonly ISet<string> _validMetricNames = new HashSet<string>(new[]
         {
-            Core.Scraping.Constants.Defaults.MessageCountMetricName,
-            Core.Scraping.Constants.Defaults.TimeSpentInQueueMetricName
+            AzureStorageConstants.Queues.Metrics.MessageCount,
+            AzureStorageConstants.Queues.Metrics.TimeSpentInQueue
         });
-        
+
         public List<string> Validate(StorageQueueMetricDefinition metricDefinition)
         {
             var errorMessages = new List<string>();
-            
+
             if (string.IsNullOrWhiteSpace(metricDefinition.AccountName))
             {
-                errorMessages.Add("No Azure Storage Account Name is configured");
+                errorMessages.Add(item: "No Azure Storage Account Name is configured");
             }
-            
+
             if (string.IsNullOrWhiteSpace(metricDefinition.QueueName))
             {
-                errorMessages.Add("No Azure Storage Queue Name is configured");
+                errorMessages.Add(item: "No Azure Storage Queue Name is configured");
             }
-            
+
             if (string.IsNullOrWhiteSpace(metricDefinition.SasToken))
             {
-                errorMessages.Add("No Azure Storage SAS Token is configured");
+                errorMessages.Add(item: "No Azure Storage SAS Token is configured");
             }
 
-            if (!_validMetricNames.Contains(metricDefinition.AzureMetricConfiguration.MetricName))
+            if (!_validMetricNames.Any(metricName => metricName.Equals(metricDefinition.AzureMetricConfiguration.MetricName, StringComparison.InvariantCultureIgnoreCase)))
             {
                 errorMessages.Add($"Invalid metric name {metricDefinition.AzureMetricConfiguration.MetricName}");
             }

src/Promitor.Scraper.Tests.Unit/Builders/MetricsDeclarationBuilder.cs

@@ -4,6 +4,7 @@
 using Promitor.Core.Scraping.Configuration.Model;
 using Promitor.Core.Scraping.Configuration.Model.Metrics.ResourceTypes;
 using Promitor.Core.Scraping.Configuration.Serialization;
+using Promitor.Integrations.AzureStorage;
 
 namespace Promitor.Scraper.Tests.Unit.Builders
 {
@@ -63,7 +64,7 @@ public string Build()
             return this;
         }
 
-        public MetricsDeclarationBuilder WithAzureStorageQueueMetric(string metricName = "promitor", string metricDescription = "Description for a metric", string queueName = "promitor-queue", string accountName = "promitor-account", string sasToken="?sig=promitor", string azureMetricName = "MessageCount")
+        public MetricsDeclarationBuilder WithAzureStorageQueueMetric(string metricName = "promitor", string metricDescription = "Description for a metric", string queueName = "promitor-queue", string accountName = "promitor-account", string sasToken="?sig=promitor", string azureMetricName = AzureStorageConstants.Queues.Metrics.MessageCount)
         {
             var azureMetricConfiguration = CreateAzureMetricConfiguration(azureMetricName);
             var metric = new StorageQueueMetricDefinition