Adding scraping interval to metricDefaults and metrics (#456)

* Adding **configuration** support for metricDefaults.scraping.inteval fixes #259 * Adding **configuration** support for <metric>.scraping.interval Fixes #258 * unit tests for scraping interval config params * adding .orig to ignored extensions due to merge conflict resolution for most merge tools * Moving ScrapingInterval -> Scraping model + Interval property so serialize/deserialize matches the YAML definition
tomkerkhove · Mar 27, 2019 · a903949 · a903949
1 parent 71244fc
commit a903949
Show file tree

Hide file tree

Showing 12 changed files with 135 additions and 42 deletions.
diff --git a/.gitignore b/.gitignore
@@ -293,3 +293,4 @@ __pycache__/
 # Local Jekyll output
 _site/*
 docs/_site/*
+*.orig
diff --git a/src/Promitor.Core.Scraping/Configuration/Model/MetricDefaults.cs b/src/Promitor.Core.Scraping/Configuration/Model/MetricDefaults.cs
@@ -3,5 +3,7 @@
     public class MetricDefaults
     {
         public Aggregation Aggregation { get; set; } = new Aggregation();
+
+        public Scraping Scraping { get; set; } = new Scraping();
     }
 }
diff --git a/src/Promitor.Core.Scraping/Configuration/Model/Metrics/MetricDefinition.cs b/src/Promitor.Core.Scraping/Configuration/Model/Metrics/MetricDefinition.cs
@@ -27,5 +27,10 @@ public abstract class MetricDefinition
         ///     Type of resource that is configured
         /// </summary>
         public abstract ResourceType ResourceType { get; }
+
+        /// <summary>
+        /// Gets or sets the scraping model.
+        /// </summary>
+        public Scraping Scraping { get; set; } = new Scraping();
     }
 }
diff --git a/src/Promitor.Core.Scraping/Configuration/Model/Scraping.cs b/src/Promitor.Core.Scraping/Configuration/Model/Scraping.cs
@@ -0,0 +1,9 @@
+using System;
+
+namespace Promitor.Core.Scraping.Configuration.Model
+{
+    public class Scraping
+    {
+        public TimeSpan? Interval { get; set; }
+    }
+}
diff --git a/src/Promitor.Core.Scraping/Configuration/Serialization/Core/MetricDefaultsDeserializer.cs b/src/Promitor.Core.Scraping/Configuration/Serialization/Core/MetricDefaultsDeserializer.cs
@@ -1,4 +1,6 @@
-using GuardNet;
+using System;
+using System.Collections.Generic;
+using GuardNet;
 using Microsoft.Extensions.Logging;
 using Promitor.Core.Scraping.Configuration.Model;
 using YamlDotNet.RepresentationModel;
@@ -15,18 +17,37 @@ internal override MetricDefaults Deserialize(YamlMappingNode node)
         {
             Guard.NotNull(node, nameof(node));
 
-            Aggregation aggregation = null;
+            var metricDefaults = new MetricDefaults();
+
             if (node.Children.ContainsKey("aggregation"))
             {
-                var metricDefaultsNode = (YamlMappingNode) node.Children[new YamlScalarNode("aggregation")];
+                var metricDefaultsNode = (YamlMappingNode)node.Children[new YamlScalarNode("aggregation")];
                 var metricDefaultsSerializer = new AggregationDeserializer(Logger);
-                aggregation = metricDefaultsSerializer.Deserialize(metricDefaultsNode);
+                var aggregation = metricDefaultsSerializer.Deserialize(metricDefaultsNode);
+                metricDefaults.Aggregation = aggregation;
             }
 
-            return new MetricDefaults
+            if (node.Children.ContainsKey(@"scraping"))
             {
-                Aggregation = aggregation
-            };
+                var scrapingNode = (YamlMappingNode)node.Children[new YamlScalarNode(@"scraping")];
+                try
+                {
+                    var scrapingIntervalNode = scrapingNode.Children[new YamlScalarNode(@"interval")];
+
+                    if (scrapingIntervalNode != null)
+                    {
+                        var scrapingIntervalTimeSpan = TimeSpan.Parse(scrapingIntervalNode.ToString());
+                        metricDefaults.Scraping.Interval = scrapingIntervalTimeSpan;
+                    }
+                }
+                catch (KeyNotFoundException)
+                {
+                    // happens when the YAML doesn't have the properties in it which is fine because the object
+                    // will get a default interval of 'null'
+                }
+            }
+
+            return metricDefaults;
         }
     }
 }
diff --git a/src/Promitor.Core.Scraping/Configuration/Serialization/Core/MetricDeserializer.cs b/src/Promitor.Core.Scraping/Configuration/Serialization/Core/MetricDeserializer.cs
@@ -1,4 +1,6 @@
-using GuardNet;
+using System;
+using System.Collections.Generic;
+using GuardNet;
 using Microsoft.Extensions.Logging;
 using Promitor.Core.Scraping.Configuration.Model.Metrics;
 using YamlDotNet.RepresentationModel;
@@ -38,6 +40,25 @@ protected virtual TMetricDefinition DeserializeMetricDefinition<TMetricDefinitio
                 AzureMetricConfiguration = azureMetricConfiguration
             };
 
+            if (metricNode.Children.ContainsKey(@"scraping"))
+            {
+                var scrapingNode = (YamlMappingNode)metricNode.Children[new YamlScalarNode(@"scraping")];
+                try
+                {
+                    var scrapingIntervalNode = scrapingNode?.Children[new YamlScalarNode(@"interval")];
+
+                    if (scrapingIntervalNode != null)
+                    {
+                        metricDefinition.Scraping.Interval = TimeSpan.Parse(scrapingIntervalNode.ToString());
+                    }
+                }
+                catch (KeyNotFoundException)
+                {
+                    // happens when the YAML doesn't have the properties in it which is fine because the object
+                    // will get a default interval of 'null'
+                }
+            }
+
             return metricDefinition;
         }
     }

diff --git a/...ation/MetricsDeclaration/MetricsDeclarationWithAzureStorageQueueYamlSerializationTests.cs b/...ation/MetricsDeclaration/MetricsDeclarationWithAzureStorageQueueYamlSerializationTests.cs
@@ -16,14 +16,14 @@ namespace Promitor.Scraper.Tests.Unit.Serialization.MetricsDeclaration
     public class MetricsDeclarationWithAzureStorageQueueYamlSerializationTests : YamlSerializationTests<StorageQueueMetricDefinition>
     {
         [Theory]
-        [InlineData("promitor1")]
-        [InlineData(null)]
-        public void YamlSerialization_SerializeAndDeserializeValidConfigForAzureStorageQueue_SucceedsWithIdenticalOutput(string resourceGroupName)
+        [InlineData("promitor1", @"01:00", @"2:00")]
+        [InlineData(null, null, null)]
+        public void YamlSerialization_SerializeAndDeserializeValidConfigForAzureStorageQueue_SucceedsWithIdenticalOutput(string resourceGroupName, string defaultScrapingInterval, string metricScrapingInterval)
         {
             // Arrange
             var azureMetadata = GenerateBogusAzureMetadata();
-            var azureStorageQueueMetricDefinition = GenerateBogusAzureStorageQueueMetricDefinition(resourceGroupName);
-            var metricDefaults = GenerateBogusMetricDefaults();
+            var azureStorageQueueMetricDefinition = GenerateBogusAzureStorageQueueMetricDefinition(resourceGroupName, metricScrapingInterval);
+            var metricDefaults = GenerateBogusMetricDefaults(defaultScrapingInterval);
             var scrapingConfiguration = new Core.Scraping.Configuration.Model.MetricsDeclaration
             {
                 AzureMetadata = azureMetadata,
@@ -64,9 +64,11 @@ private static void AssertAzureStorageQueueMetricDefinition(StorageQueueMetricDe
             Assert.Equal(serviceBusMetricDefinition.AzureMetricConfiguration.Aggregation.Interval, deserializedMetricDefinition.AzureMetricConfiguration.Aggregation.Interval);
         }
 
-        private StorageQueueMetricDefinition GenerateBogusAzureStorageQueueMetricDefinition(string resourceGroupName)
+        private StorageQueueMetricDefinition GenerateBogusAzureStorageQueueMetricDefinition(string resourceGroupName, string metricScrapingInterval)
         {
+            var bogusScrapingInterval = GenerateBogusScrapingInterval(metricScrapingInterval);
             var bogusAzureMetricConfiguration = GenerateBogusAzureMetricConfiguration();
+
             var bogusGenerator = new Faker<StorageQueueMetricDefinition>()
                 .StrictMode(ensureRulesForAllProperties: true)
                 .RuleFor(metricDefinition => metricDefinition.Name, faker => faker.Name.FirstName())
@@ -77,6 +79,7 @@ private StorageQueueMetricDefinition GenerateBogusAzureStorageQueueMetricDefinit
                 .RuleFor(metricDefinition => metricDefinition.SasToken, faker => $"?sig={Base64UrlEncoder.Encode(faker.Lorem.Sentence(wordCount: 3))}")
                 .RuleFor(metricDefinition => metricDefinition.AzureMetricConfiguration, faker => bogusAzureMetricConfiguration)
                 .RuleFor(metricDefinition => metricDefinition.ResourceGroupName, faker => resourceGroupName)
+                .RuleFor(metricDefinition => metricDefinition.Scraping, faker => bogusScrapingInterval)
                 .Ignore(metricDefinition => metricDefinition.ResourceGroupName);
 
             return bogusGenerator.Generate();

diff --git a/...ation/MetricsDeclaration/MetricsDeclarationWithContainerInstanceYamlSerializationTests.cs b/...ation/MetricsDeclaration/MetricsDeclarationWithContainerInstanceYamlSerializationTests.cs
@@ -15,14 +15,14 @@ namespace Promitor.Scraper.Tests.Unit.Serialization.MetricsDeclaration
     public class MetricsDeclarationWithContainerInstanceYamlSerializationTests : YamlSerializationTests<ContainerInstanceMetricDefinition>
     {
         [Theory]
-        [InlineData("promitor1")]
-        [InlineData(null)]
-        public void YamlSerialization_SerializeAndDeserializeValidConfigForContainerInstance_SucceedsWithIdenticalOutput(string resourceGroupName)
+        [InlineData("promitor1", @"01:00", @"2:00")]
+        [InlineData(null, null, null)]
+        public void YamlSerialization_SerializeAndDeserializeValidConfigForContainerInstance_SucceedsWithIdenticalOutput(string resourceGroupName, string defaultScrapingInterval, string metricScrapingInterval)
         {
             // Arrange
             var azureMetadata = GenerateBogusAzureMetadata();
-            var containerInstanceMetricDefinition = GenerateBogusContainerInstanceMetricDefinition(resourceGroupName);
-            var metricDefaults = GenerateBogusMetricDefaults();
+            var containerInstanceMetricDefinition = GenerateBogusContainerInstanceMetricDefinition(resourceGroupName, metricScrapingInterval);
+            var metricDefaults = GenerateBogusMetricDefaults(defaultScrapingInterval);
             var scrapingConfiguration = new Core.Scraping.Configuration.Model.MetricsDeclaration
             {
                 AzureMetadata = azureMetadata,
@@ -60,9 +60,11 @@ private static void AssertContainerInstanceMetricDefinition(ContainerInstanceMet
             Assert.Equal(containerInstanceMetricDefinition.AzureMetricConfiguration.Aggregation.Type, deserializedMetricDefinition.AzureMetricConfiguration.Aggregation.Type);
             Assert.Equal(containerInstanceMetricDefinition.AzureMetricConfiguration.Aggregation.Interval, deserializedMetricDefinition.AzureMetricConfiguration.Aggregation.Interval);
         }
-        private ContainerInstanceMetricDefinition GenerateBogusContainerInstanceMetricDefinition(string resourceGroupName)
+        private ContainerInstanceMetricDefinition GenerateBogusContainerInstanceMetricDefinition(string resourceGroupName, string metricScrapingInterval)
         {
+            var bogusScrapingInterval = GenerateBogusScrapingInterval(metricScrapingInterval);
             var bogusAzureMetricConfiguration = GenerateBogusAzureMetricConfiguration();
+
             var bogusGenerator = new Faker<ContainerInstanceMetricDefinition>()
                 .StrictMode(ensureRulesForAllProperties: true)
                 .RuleFor(metricDefinition => metricDefinition.Name, faker => faker.Name.FirstName())
@@ -71,6 +73,7 @@ private ContainerInstanceMetricDefinition GenerateBogusContainerInstanceMetricDe
                 .RuleFor(metricDefinition => metricDefinition.ContainerGroup, faker => faker.Name.LastName())
                 .RuleFor(metricDefinition => metricDefinition.AzureMetricConfiguration, faker => bogusAzureMetricConfiguration)
                 .RuleFor(metricDefinition => metricDefinition.ResourceGroupName, faker => resourceGroupName)
+                .RuleFor(metricDefinition => metricDefinition.Scraping, faker => bogusScrapingInterval)
                 .Ignore(metricDefinition => metricDefinition.ResourceGroupName);
 
             return bogusGenerator.Generate();

diff --git a/...ation/MetricsDeclaration/MetricsDeclarationWithContainerRegistryYamlSerializationTests.cs b/...ation/MetricsDeclaration/MetricsDeclarationWithContainerRegistryYamlSerializationTests.cs
@@ -15,14 +15,14 @@ namespace Promitor.Scraper.Tests.Unit.Serialization.MetricsDeclaration
     public class MetricsDeclarationWithContainerRegistryYamlSerializationTests : YamlSerializationTests<ContainerRegistryMetricDefinition>
     {
         [Theory]
-        [InlineData("promitor1")]
-        [InlineData(null)]
-        public void YamlSerialization_SerializeAndDeserializeValidConfigForContainerRegistry_SucceedsWithIdenticalOutput(string resourceGroupName)
+        [InlineData("promitor1", @"01:00", @"2:00")]
+        [InlineData(null, null, null)]
+        public void YamlSerialization_SerializeAndDeserializeValidConfigForContainerRegistry_SucceedsWithIdenticalOutput(string resourceGroupName, string defaultScrapingInterval, string metricScrapingInterval)
         {
             // Arrange
             var azureMetadata = GenerateBogusAzureMetadata();
-            var containerRegistryMetricDefinition = GenerateBogusContainerRegistryMetricDefinition(resourceGroupName);
-            var metricDefaults = GenerateBogusMetricDefaults();
+            var containerRegistryMetricDefinition = GenerateBogusContainerRegistryMetricDefinition(resourceGroupName, metricScrapingInterval);
+            var metricDefaults = GenerateBogusMetricDefaults(defaultScrapingInterval);
             var scrapingConfiguration = new Core.Scraping.Configuration.Model.MetricsDeclaration
             {
                 AzureMetadata = azureMetadata,
@@ -56,9 +56,11 @@ private static void AssertContainerRegistryMetricDefinition(ContainerRegistryMet
             Assert.Equal(serviceBusMetricDefinition.RegistryName, deserializedServiceBusMetricDefinition.RegistryName);
         }
 
-        private ContainerRegistryMetricDefinition GenerateBogusContainerRegistryMetricDefinition(string resourceGroupName)
+        private ContainerRegistryMetricDefinition GenerateBogusContainerRegistryMetricDefinition(string resourceGroupName, string metricScrapingInterval)
         {
+            var bogusScrapingInterval = GenerateBogusScrapingInterval(metricScrapingInterval);
             var bogusAzureMetricConfiguration = GenerateBogusAzureMetricConfiguration();
+
             var bogusGenerator = new Faker<ContainerRegistryMetricDefinition>()
                 .StrictMode(ensureRulesForAllProperties: true)
                 .RuleFor(metricDefinition => metricDefinition.Name, faker => faker.Name.FirstName())
@@ -67,6 +69,7 @@ private ContainerRegistryMetricDefinition GenerateBogusContainerRegistryMetricDe
                 .RuleFor(metricDefinition => metricDefinition.RegistryName, faker => faker.Name.LastName())
                 .RuleFor(metricDefinition => metricDefinition.AzureMetricConfiguration, faker => bogusAzureMetricConfiguration)
                 .RuleFor(metricDefinition => metricDefinition.ResourceGroupName, faker => resourceGroupName)
+                .RuleFor(metricDefinition => metricDefinition.Scraping, faker => bogusScrapingInterval)
                 .Ignore(metricDefinition => metricDefinition.ResourceGroupName);
 
             return bogusGenerator.Generate();

diff --git a/...ization/MetricsDeclaration/MetricsDeclarationWithServiceBusQueueYamlSerializationTests.cs b/...ization/MetricsDeclaration/MetricsDeclarationWithServiceBusQueueYamlSerializationTests.cs
@@ -15,14 +15,14 @@ namespace Promitor.Scraper.Tests.Unit.Serialization.MetricsDeclaration
     public class ServiceBusQueueYamlSerializationTests : YamlSerializationTests<ServiceBusQueueMetricDefinition>
     {
         [Theory]
-        [InlineData("promitor1")]
-        [InlineData(null)]
-        public void YamlSerialization_SerializeAndDeserializeValidConfigForServiceBus_SucceedsWithIdenticalOutput(string resourceGroupName)
+        [InlineData("promitor1", @"01:00", @"2:00")]
+        [InlineData(null, null, null)]
+        public void YamlSerialization_SerializeAndDeserializeValidConfigForServiceBus_SucceedsWithIdenticalOutput(string resourceGroupName, string defaultScrapingInterval, string metricScrapingInterval)
         {
             // Arrange
             var azureMetadata = GenerateBogusAzureMetadata();
-            var serviceBusMetricDefinition = GenerateBogusServiceBusMetricDefinition(resourceGroupName);
-            var metricDefaults = GenerateBogusMetricDefaults();
+            var serviceBusMetricDefinition = GenerateBogusServiceBusMetricDefinition(resourceGroupName, metricScrapingInterval);
+            var metricDefaults = GenerateBogusMetricDefaults(defaultScrapingInterval);
             var scrapingConfiguration = new Core.Scraping.Configuration.Model.MetricsDeclaration
             {
                 AzureMetadata = azureMetadata,
@@ -57,9 +57,11 @@ private static void AssertServiceBusQueueMetricDefinition(ServiceBusQueueMetricD
             Assert.Equal(serviceBusMetricDefinition.QueueName, deserializedServiceBusMetricDefinition.QueueName);
         }
 
-        private ServiceBusQueueMetricDefinition GenerateBogusServiceBusMetricDefinition(string resourceGroupName)
+        private ServiceBusQueueMetricDefinition GenerateBogusServiceBusMetricDefinition(string resourceGroupName, string metricScrapingInterval)
         {
+            var bogusScrapingInterval = GenerateBogusScrapingInterval(metricScrapingInterval);
             var bogusAzureMetricConfiguration = GenerateBogusAzureMetricConfiguration();
+
             var bogusGenerator = new Faker<ServiceBusQueueMetricDefinition>()
                 .StrictMode(ensureRulesForAllProperties: true)
                 .RuleFor(metricDefinition => metricDefinition.Name, faker => faker.Name.FirstName())
@@ -69,7 +71,8 @@ private ServiceBusQueueMetricDefinition GenerateBogusServiceBusMetricDefinition(
                 .RuleFor(metricDefinition => metricDefinition.QueueName, faker => faker.Name.FirstName())
                 .RuleFor(metricDefinition => metricDefinition.AzureMetricConfiguration, faker => bogusAzureMetricConfiguration)
                 .RuleFor(metricDefinition => metricDefinition.ResourceGroupName, faker => resourceGroupName)
-                .Ignore(metricDefinition  => metricDefinition.ResourceGroupName);
+                .RuleFor(metricDefinition => metricDefinition.Scraping, faker => bogusScrapingInterval)
+                .Ignore(metricDefinition => metricDefinition.ResourceGroupName);
 
             return bogusGenerator.Generate();
         }
-Original file line number
+Diff line change
@@ Expand Up / @@ -293,3 +293,4 @@ __pycache__/ @@
     # Local Jekyll output
     _site/*
     docs/_site/*
+    *.orig