Network interface metrics (#466)

* Adjust .gitignore to exclude .DS_Store

* Add config doc for network interfaces.

* Add Network Interface Resource Definition

* Add Network interface deserializer

* Adjust index

* Add Network Interface to Deserializer Factory

* Add Network Interface Scraper support

* Add validator.

* Bugfix to Network interface scraper.

* Add unit test for Network Interface.

* Fix doc typo

* Add link to Microsoft docs for azure network interface.

* I hate you git

.gitignore

@@ -293,4 +293,8 @@ __pycache__/
 # Local Jekyll output
 _site/*
 docs/_site/*
-*.orig
+
+#MAC
+.DS_Store
+
+*.orig

docs/configuration/metrics/index.md

@@ -16,7 +16,7 @@ azureMetadata:
 metricDefaults:
   aggregation:
     interval: 00:05:00
-metrics: 
+metrics:
   - name: demo_queue_size
     description: "Amount of active messages of the 'myqueue' queue"
     resourceType: ServiceBusQueue
@@ -74,6 +74,7 @@ We also provide a simplified way to configure the following Azure resources:
 - [Azure Container Registry](container-registry)
 - [Azure Service Bus Queue](service-bus-queue)
 - [Azure Virtual Machine](virtual-machine)
+- [Azure Network Interface](network-interface)
 - [Azure Storage Queue](storage-queue)
 
 Want to help out? Create an issue and [contribute a new scraper](https://github.com/tomkerkhove/promitor/blob/master/adding-a-new-scraper.md).

docs/configuration/metrics/network-interfaces.md

@@ -0,0 +1,27 @@
+---
+layout: default
+title: Azure Network Interface Declaration
+---
+
+## Azure Network Interface
+You can declare to scrape an [Azure Network Interface](https://docs.microsoft.com/en-us/azure/virtual-network/virtual-network-network-interface) via the `NetworkInterface` resource type.
+
+The following fields need to be provided:
+- `networkInterfaceName` - The name of the network interface
+
+All supported metrics are documented in the official [Azure Monitor documentation](https://docs.microsoft.com/en-us/azure/azure-monitor/platform/metrics-supported#microsoftnetworknetworkinterfaces).
+
+Example:
+```yaml
+  - name: demo_azuresnetworkinterface_bytesreceivedrate
+    description: "Number of bytes the Network Interface sent"
+    resourceType: NetworkInterface
+    networkInterfaceName: promitor-network-interface
+    azureMetricConfiguration:
+      metricName: BytesReceivedRate
+      aggregation:
+        type: Average
+```
+
+[&larr; back to metrics declarations](/configuration/metrics)<br />
+[&larr; back to introduction](/)

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/NetworkInterfaceMetricDefinition.cs

@@ -0,0 +1,8 @@
+namespace Promitor.Core.Scraping.Configuration.Model.Metrics.ResourceTypes
+{
+    public class NetworkInterfaceMetricDefinition : MetricDefinition
+    {
+        public string NetworkInterfaceName { get; set; }
+        public override ResourceType ResourceType { get; } = ResourceType.NetworkInterface;
+    }
+}

src/Promitor.Core.Scraping/Configuration/Model/ResourceType.cs

@@ -8,6 +8,7 @@ public enum ResourceType
         StorageQueue = 3,
         ContainerInstance = 4,
         VirtualMachine = 5,
-        ContainerRegistry = 6
+        ContainerRegistry = 6,
+        NetworkInterface = 7,
     }
 }

src/Promitor.Core.Scraping/Configuration/Serialization/Deserializers/NetworkInterfaceMetricDeserializer.cs

@@ -0,0 +1,22 @@
+using Promitor.Core.Scraping.Configuration.Model.Metrics;
+using Promitor.Core.Scraping.Configuration.Model.Metrics.ResourceTypes;
+using YamlDotNet.RepresentationModel;
+
+namespace Promitor.Core.Scraping.Configuration.Serialization.Deserializers
+{
+    internal class NetworkInterfaceMetricDeserializer : GenericAzureMetricDeserializer
+    {
+        /// <summary>Deserializes the specified Network Interface metric node from the YAML configuration file.</summary>
+        /// <param name="metricNode">The metric node containing 'networkInterfaceName' parameter pointing to an instance of a Network Interface</param>
+        /// <returns>A new <see cref="MetricDefinition"/> object (strongly typed as a <see cref="NetworkInterfaceMetricDefinition"/>) </returns>
+        internal override MetricDefinition Deserialize(YamlMappingNode metricNode)
+        {
+            var metricDefinition = base.DeserializeMetricDefinition<NetworkInterfaceMetricDefinition>(metricNode);
+            var networkInterfaceName = metricNode.Children[new YamlScalarNode("networkInterfaceName")];
+
+            metricDefinition.NetworkInterfaceName = networkInterfaceName?.ToString();
+
+            return metricDefinition;
+        }
+    }
+}

src/Promitor.Core.Scraping/Factories/MetricDeserializerFactory.cs

@@ -21,6 +21,8 @@ internal static GenericAzureMetricDeserializer GetDeserializerFor(Configuration.
                     return new VirtualMachineMetricDeserializer();
                 case Configuration.Model.ResourceType.ContainerRegistry:
                     return new ContainerRegistryMetricDeserializer();
+                case Configuration.Model.ResourceType.NetworkInterface:
+                    return new NetworkInterfaceMetricDeserializer();
             }
 
             throw new ArgumentOutOfRangeException($@"Resource Type {resource} not supported.");

src/Promitor.Core.Scraping/Factories/MetricScraperFactory.cs

@@ -37,6 +37,8 @@ public static IScraper<MetricDefinition> CreateScraper(ResourceType metricDefini
                     return new ContainerInstanceScraper(azureMetadata, metricDefaults, azureMonitorClient, logger, exceptionTracker);
                 case ResourceType.VirtualMachine:
                     return new VirtualMachineScraper(azureMetadata, metricDefaults, azureMonitorClient, logger, exceptionTracker);
+                case ResourceType.NetworkInterface:
+                    return new NetworkInterfaceScraper(azureMetadata, metricDefaults, azureMonitorClient, logger, exceptionTracker);
                 case ResourceType.ContainerRegistry:
                     return new ContainerRegistryScraper(azureMetadata, metricDefaults, azureMonitorClient, logger, exceptionTracker);
                 default:

src/Promitor.Core.Scraping/ResourceTypes/NetworkInterfaceScraper.cs

@@ -0,0 +1,31 @@
+using System;
+using System.Threading.Tasks;
+using Microsoft.Azure.Management.Monitor.Fluent.Models;
+using Microsoft.Extensions.Logging;
+using Promitor.Core.Scraping.Configuration.Model;
+using Promitor.Core.Scraping.Configuration.Model.Metrics.ResourceTypes;
+using Promitor.Core.Telemetry.Interfaces;
+using Promitor.Integrations.AzureMonitor;
+
+namespace Promitor.Core.Scraping.ResourceTypes
+{
+    internal class NetworkInterfaceScraper : Scraper<NetworkInterfaceMetricDefinition>
+    {
+        private const string ResourceUriTemplate = "subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.Network/networkInterfaces/{2}";
+
+        public NetworkInterfaceScraper(AzureMetadata azureMetadata, MetricDefaults metricDefaults, AzureMonitorClient azureMonitorClient, ILogger logger, IExceptionTracker exceptionTracker)
+            : base(azureMetadata, metricDefaults, azureMonitorClient, logger, exceptionTracker)
+        {
+        }
+
+        protected override async Task<double> ScrapeResourceAsync(string subscriptionId, string resourceGroupName, NetworkInterfaceMetricDefinition metricDefinition, AggregationType aggregationType, TimeSpan aggregationInterval)
+        {
+            var resourceUri = string.Format(ResourceUriTemplate, AzureMetadata.SubscriptionId, AzureMetadata.ResourceGroupName, metricDefinition.NetworkInterfaceName);
+
+            var metricName = metricDefinition.AzureMetricConfiguration.MetricName;
+            var foundMetricValue = await AzureMonitorClient.QueryMetricAsync(metricName, aggregationType, aggregationInterval, resourceUri);
+
+            return foundMetricValue;
+        }
+    }
+}

src/Promitor.Scraper.Host/Validation/Factories/MetricValidatorFactory.cs

@@ -21,6 +21,8 @@ internal static IMetricValidator GetValidatorFor(ResourceType resourceType)
                     return new ContainerInstanceMetricValidator();
                 case ResourceType.VirtualMachine:
                     return new VirtualMachineMetricValidator();
+                case ResourceType.NetworkInterface:
+                    return new NetworkInterfaceMetricValidator();
                 case ResourceType.ContainerRegistry:
                     return new ContainerRegistryMetricValidator();
             }

src/Promitor.Scraper.Host/Validation/MetricDefinitions/ResourceTypes/NetworkInterfaceMetricValidator.cs

@@ -0,0 +1,23 @@
+using System.Collections.Generic;
+using GuardNet;
+using Promitor.Core.Scraping.Configuration.Model.Metrics.ResourceTypes;
+
+namespace Promitor.Scraper.Host.Validation.MetricDefinitions.ResourceTypes
+{
+    internal class NetworkInterfaceMetricValidator : MetricValidator<NetworkInterfaceMetricDefinition>
+    {
+        protected override IEnumerable<string> Validate(NetworkInterfaceMetricDefinition networkInterfaceMetricDefinition)
+        {
+            Guard.NotNull(networkInterfaceMetricDefinition, nameof(networkInterfaceMetricDefinition));
+
+            var errorMessages = new List<string>();
+
+            if (string.IsNullOrWhiteSpace(networkInterfaceMetricDefinition.NetworkInterfaceName))
+            {
+                errorMessages.Add("No network interface name is configured");
+            }
+
+            return errorMessages;
+        }
+    }
+}

src/Promitor.Scraper.Tests.Unit/Builders/MetricsDeclarationBuilder.cs

@@ -127,6 +127,22 @@ public string Build()
             return this;
         }
 
+        public MetricsDeclarationBuilder WithNetworkInterfaceMetric(string metricName = "promitor-network-interface", string metricDescription = "Description for a metric", string networkInterfaceName = "promitor-network-interface-name",  string azureMetricName = "Total")
+        {
+            var azureMetricConfiguration = CreateAzureMetricConfiguration(azureMetricName);
+            var metric = new NetworkInterfaceMetricDefinition
+            {
+                Name = metricName,
+                Description = metricDescription,
+                NetworkInterfaceName = networkInterfaceName,
+                AzureMetricConfiguration = azureMetricConfiguration
+            };
+
+            _metrics.Add(metric);
+
+            return this;
+        }
+
         public MetricsDeclarationBuilder WithGenericMetric(string metricName = "foo", string metricDescription = "Description for a metric", string resourceUri = "Microsoft.ServiceBus/namespaces/promitor-messaging", string filter = "EntityName eq \'orders\'", string azureMetricName = "Total")
         {
             var azureMetricConfiguration = CreateAzureMetricConfiguration(azureMetricName);

src/Promitor.Scraper.Tests.Unit/Serialization/MetricsDeclaration/MetricsDeclarationWithNetworkInterfaceYamlSerializationTests.cs

@@ -0,0 +1,77 @@
+using System.Collections.Generic;
+using System.ComponentModel;
+using System.Linq;
+using Bogus;
+using Microsoft.Extensions.Logging.Abstractions;
+using Promitor.Core.Scraping.Configuration.Model;
+using Promitor.Core.Scraping.Configuration.Model.Metrics;
+using Promitor.Core.Scraping.Configuration.Model.Metrics.ResourceTypes;
+using Promitor.Core.Scraping.Configuration.Serialization.Core;
+using Xunit;
+
+namespace Promitor.Scraper.Tests.Unit.Serialization.MetricsDeclaration
+{
+    [Category(category: "Unit")]
+    public class MetricsDeclarationWithNetworkInterfaceYamlSerializationTests : YamlSerializationTests<NetworkInterfaceMetricDefinition>
+    {
+        [Theory]
+        [InlineData("promitor1", @"01:00", @"2:00")]
+        [InlineData(null, null, null)]
+        public void YamlSerialization_SerializeAndDeserializeValidConfigForNetworkInterface_SucceedsWithIdenticalOutput(string resourceGroupName, string defaultScrapingInterval, string metricScrapingInterval)
+        {
+            // Arrange
+            var azureMetadata = GenerateBogusAzureMetadata();
+            var networkInterfaceMetricDefinition = GenerateBogusNetworkInterfaceMetricDefinition(resourceGroupName, metricScrapingInterval);
+            var metricDefaults = GenerateBogusMetricDefaults(defaultScrapingInterval);
+            var scrapingConfiguration = new Core.Scraping.Configuration.Model.MetricsDeclaration
+            {
+                AzureMetadata = azureMetadata,
+                MetricDefaults = metricDefaults,
+                Metrics = new List<MetricDefinition>
+                {
+                    networkInterfaceMetricDefinition
+                }
+            };
+            var configurationSerializer = new ConfigurationSerializer(NullLogger.Instance);
+
+            // Act
+            var serializedConfiguration = configurationSerializer.Serialize(scrapingConfiguration);
+            var deserializedConfiguration = configurationSerializer.Deserialize(serializedConfiguration);
+
+            // Assert
+            Assert.NotNull(deserializedConfiguration);
+            AssertAzureMetadata(deserializedConfiguration, azureMetadata);
+            AssertMetricDefaults(deserializedConfiguration, metricDefaults);
+            Assert.NotNull(deserializedConfiguration.Metrics);
+            Assert.Single(deserializedConfiguration.Metrics);
+            var deserializedMetricDefinition = deserializedConfiguration.Metrics.FirstOrDefault();
+            AssertMetricDefinition(deserializedMetricDefinition, networkInterfaceMetricDefinition);
+            var deserializedNetworkInterfaceMetricDefinition = deserializedMetricDefinition as NetworkInterfaceMetricDefinition;
+            AssertNetworkInterfaceMetricDefinition(deserializedNetworkInterfaceMetricDefinition, networkInterfaceMetricDefinition);
+        }
+
+        private static void AssertNetworkInterfaceMetricDefinition(NetworkInterfaceMetricDefinition deserializedNetworkInterfaceMetricDefinition, NetworkInterfaceMetricDefinition networkInterfaceMetricDefinition)
+        {
+            Assert.NotNull(deserializedNetworkInterfaceMetricDefinition);
+            Assert.Equal(networkInterfaceMetricDefinition.NetworkInterfaceName, deserializedNetworkInterfaceMetricDefinition.NetworkInterfaceName);
+        }
+
+        private NetworkInterfaceMetricDefinition GenerateBogusNetworkInterfaceMetricDefinition(string resourceGroupName, string metricScrapingInterval)
+        {
+            var bogusScrapingInterval = GenerateBogusScrapingInterval(metricScrapingInterval);
+            var bogusAzureMetricConfiguration = GenerateBogusAzureMetricConfiguration();
+            Faker<NetworkInterfaceMetricDefinition> bogusGenerator = new Faker<NetworkInterfaceMetricDefinition>()
+                .StrictMode(ensureRulesForAllProperties: true)
+                .RuleFor(metricDefinition => metricDefinition.Name, faker => faker.Name.FirstName())
+                .RuleFor(metricDefinition => metricDefinition.Description, faker => faker.Lorem.Sentence(wordCount: 6))
+                .RuleFor(metricDefinition => metricDefinition.ResourceType, faker => ResourceType.NetworkInterface)
+                .RuleFor(metricDefinition => metricDefinition.NetworkInterfaceName, faker => faker.Name.LastName())
+                .RuleFor(metricDefinition => metricDefinition.AzureMetricConfiguration, faker => bogusAzureMetricConfiguration)
+                .RuleFor(metricDefinition => metricDefinition.ResourceGroupName, faker => resourceGroupName)
+                .RuleFor(metricDefinition => metricDefinition.Scraping, faker => bogusScrapingInterval)
+                .Ignore(metricDefinition => metricDefinition.ResourceGroupName);
+
+            return bogusGenerator.Generate();
+        }
+    }
+}

src/Promitor.Scraper.Tests.Unit/Validation/Metrics/ResourceTypes/NetworkInterfaceMetricsDeclarationValidationStepsTests.cs

@@ -0,0 +1,97 @@
+using System.ComponentModel;
+using Promitor.Scraper.Host.Validation.Steps;
+using Promitor.Scraper.Tests.Unit.Builders;
+using Promitor.Scraper.Tests.Unit.Stubs;
+using Xunit;
+
+namespace Promitor.Scraper.Tests.Unit.Validation.Metrics.ResourceTypes
+{
+    [Category("Unit")]
+    public class NetworkInterfaceMetricsDeclarationValidationStepTests
+    {
+        [Fact]
+        public void NetworkInterfaceMetricsDeclaration_DeclarationWithoutAzureMetricName_Succeeds()
+        {
+            // Arrange
+            var rawDeclaration = MetricsDeclarationBuilder.WithMetadata()
+                .WithNetworkInterfaceMetric(azureMetricName: string.Empty)
+                .Build();
+            var metricsDeclarationProvider = new MetricsDeclarationProviderStub(rawDeclaration);
+
+            // Act
+            var scrapingScheduleValidationStep = new MetricsDeclarationValidationStep(metricsDeclarationProvider);
+            var validationResult = scrapingScheduleValidationStep.Run();
+
+            // Assert
+            Assert.False(validationResult.IsSuccessful, "Validation is successful");
+        }
+
+        [Fact]
+        public void NetworkInterfaceMetricsDeclaration_DeclarationWithoutMetricDescription_Succeeded()
+        {
+            // Arrange
+            var rawDeclaration = MetricsDeclarationBuilder.WithMetadata()
+                .WithNetworkInterfaceMetric(metricDescription: string.Empty)
+                .Build();
+            var metricsDeclarationProvider = new MetricsDeclarationProviderStub(rawDeclaration);
+
+            // Act
+            var scrapingScheduleValidationStep = new MetricsDeclarationValidationStep(metricsDeclarationProvider);
+            var validationResult = scrapingScheduleValidationStep.Run();
+
+            // Assert
+            Assert.True(validationResult.IsSuccessful, "Validation was not successful");
+        }
+
+        [Fact]
+        public void NetworkInterfaceMetricsDeclaration_DeclarationWithoutMetricName_Fails()
+        {
+            // Arrange
+            var rawDeclaration = MetricsDeclarationBuilder.WithMetadata()
+                .WithNetworkInterfaceMetric(string.Empty)
+                .Build();
+            var metricsDeclarationProvider = new MetricsDeclarationProviderStub(rawDeclaration);
+
+            // Act
+            var scrapingScheduleValidationStep = new MetricsDeclarationValidationStep(metricsDeclarationProvider);
+            var validationResult = scrapingScheduleValidationStep.Run();
+
+            // Assert
+            Assert.False(validationResult.IsSuccessful, "Validation is successful");
+        }
+
+        [Fact]
+        public void NetworkInterfaceMetricsDeclaration_DeclarationWithoutNetworkInterfaceName_Fails()
+        {
+            // Arrange
+            var rawDeclaration = MetricsDeclarationBuilder.WithMetadata()
+                .WithNetworkInterfaceMetric(networkInterfaceName: string.Empty)
+                .Build();
+            var metricsDeclarationProvider = new MetricsDeclarationProviderStub(rawDeclaration);
+
+            // Act
+            var scrapingScheduleValidationStep = new MetricsDeclarationValidationStep(metricsDeclarationProvider);
+            var validationResult = scrapingScheduleValidationStep.Run();
+
+            // Assert
+            Assert.False(validationResult.IsSuccessful, "Validation is successful");
+        }
+
+        [Fact]
+        public void NetworkInterfaceMetricsDeclaration_ValidDeclaration_Succeeds()
+        {
+            // Arrange
+            var rawMetricsDeclaration = MetricsDeclarationBuilder.WithMetadata()
+                .WithNetworkInterfaceMetric()
+                .Build();
+            var metricsDeclarationProvider = new MetricsDeclarationProviderStub(rawMetricsDeclaration);
+
+            // Act
+            var scrapingScheduleValidationStep = new MetricsDeclarationValidationStep(metricsDeclarationProvider);
+            var validationResult = scrapingScheduleValidationStep.Run();
+
+            // Assert
+            Assert.True(validationResult.IsSuccessful, "Validation was not successful");
+        }
+    }
+}