Provide scraper for Azure Kubernetes Service

changelog/content/experimental/unreleased.md

@@ -15,6 +15,7 @@ version:
 - {{% tag added %}} Support for scraping Azure Express Route circuits ([docs](https://promitor.io/configuration/v2.x/metrics/express-route-circuit) | [#1251](https://github.com/tomkerkhove/promitor/issues/1251) | Contributed by [@bluepixbe](https://github.com/bluepixbe) 🎉).
 - {{% tag added %}} Support for scraping Azure Application Gateway ([docs](https://promitor.io/configuration/v2.x/metrics/application-gateway) | [#1251](https://github.com/tomkerkhove/promitor/issues/313) | Contributed by [@bluepixbe](https://github.com/bluepixbe) 🎉).
 - {{% tag added %}} Support for scraping Azure Network Gateway ([docs](https://promitor.io/configuration/v2.x/metrics/network-gateway) | [#1264](https://github.com/tomkerkhove/promitor/issues/1264) | Contributed by [@bluepixbe](https://github.com/bluepixbe) 🎉).
+- {{% tag added %}} Support for scraping Azure Kubernetes Service ([docs](https://promitor.io/configuration/v2.x/metrics/kubernetes) | [#333](https://github.com/tomkerkhove/promitor/issues/333) | Contributed by [@jkataja](https://github.com/jkataja) 🎉).
 - {{% tag added %}} New validation rule to ensure at least one resource or resource collection is configured to scrape
 - {{% tag added %}} Provide suggestions when unknown fields are found in the metrics config. ([#1105](https://github.com/tomkerkhove/promitor/issues/1105) | Contributed by [@adamconnelly](https://github.com/adamconnelly) 🎉).
 - {{% tag added %}} Add validation to ensure the scraping schedule is a valid Cron expression. ([#1103](https://github.com/tomkerkhove/promitor/issues/1103) | Contributed by [@adamconnelly](https://github.com/adamconnelly) 🎉).

docs/configuration/v2.x/metrics/index.md

@@ -134,6 +134,7 @@ We also provide a simplified way to scrape the following Azure resources:
 - [Azure IoT Hub](iot-hub)
 - [Azure IoT Hub Device Provisioning Service (DPS)](iot-hub-device-provisioning-service)
 - [Azure Key Vault](key-vault)
+- [Azure Kubernetes Service](kubernetes)
 - [Azure Logic Apps](logic-apps)
 - [Azure Network Gateway](network-gateway)
 - [Azure Network Interface](network-interface)

docs/configuration/v2.x/metrics/kuberbetes.md

@@ -0,0 +1,36 @@
+---
+layout: default
+title: Azure Kubernetes Service Declaration
+---
+
+## Azure Kubernetes Service
+
+![Availability Badge](https://img.shields.io/badge/Available%20Starting-v2.0-green.svg)![Resource Discovery Support Badge](https://img.shields.io/badge/Support%20for%20Resource%20Discovery-Yes-green.svg)
+
+You can declare to scrape an Azure Kubernetes Service (AKS)
+via the `KubernetesService` resource type.
+
+The following fields need to be provided:
+
+- `clusterName` - The name of the Azure Kubernetes Service
+
+All supported metrics are documented in the official [Azure Monitor documentation](https://docs.microsoft.com/en-us/azure/azure-monitor/platform/metrics-supported#microsoftcontainerservicemanagedclusters).
+
+Example:
+
+```yaml
+name: azure_kubernetes_available_cpu_cores
+description: "Available CPU cores in cluster"
+resourceType: KubernetesService
+azureMetricConfiguration:
+  metricName: kube_node_status_allocatable_cpu_cores
+  aggregation:
+    type: Average
+resources:
+- clusterName: promitor-aks
+```
+
+<!-- markdownlint-disable MD033 -->
+[&larr; back to metrics declarations](/configuration/v2.x/metrics)<br />
+[&larr; back to introduction](/)
+<!-- markdownlint-enable -->

docs/configuration/v2.x/resource-discovery.md

@@ -101,6 +101,7 @@ Dynamic resource discovery is supported for the following scrapers:
 - [Azure IoT Hub](metrics/iot-hub)
 - [Azure IoT Hub Device Provisioning Service (DPS)](metrics/iot-hub-device-provisioning-service)
 - [Azure Key Vault](metrics/key-vault)
+- [Azure Kubernetes Service](metrics/kubernetes)
 - [Azure Logic Apps](metrics/logic-apps)
 - [Azure Network Gateway](metrics/network-gateway)
 - [Azure Network Interface](metrics/network-interface)

src/Promitor.Agents.ResourceDiscovery/Graph/ResourceDiscoveryFactory.cs

@@ -34,6 +34,8 @@ public static ResourceDiscoveryQuery UseResourceDiscoveryFor(ResourceType resour
                     return new IoTHubDiscoveryQuery();
                 case ResourceType.KeyVault:
                     return new KeyVaultDiscoveryQuery();
+                case ResourceType.KubernetesService:
+                    return new KubernetesServiceDiscoveryQuery();
                 case ResourceType.LogicApp:
                     return new LogicAppDiscoveryQuery();
                 case ResourceType.NetworkGateway:

src/Promitor.Agents.ResourceDiscovery/Graph/ResourceTypes/KubernetesServiceDiscoveryQuery.cs

@@ -0,0 +1,21 @@
+using GuardNet;
+using Newtonsoft.Json.Linq;
+using Promitor.Core.Contracts;
+using Promitor.Core.Contracts.ResourceTypes;
+
+namespace Promitor.Agents.ResourceDiscovery.Graph.ResourceTypes
+{
+    public class KubernetesServiceDiscoveryQuery : ResourceDiscoveryQuery
+    {
+        public override string[] ResourceTypes => new[] { "Microsoft.ContainerService/managedClusters" };
+        public override string[] ProjectedFieldNames => new[] { "subscriptionId", "resourceGroup", "type", "name" };
+
+        public override AzureResourceDefinition ParseResults(JToken resultRowEntry)
+        {
+            Guard.NotNull(resultRowEntry, nameof(resultRowEntry));
+
+            var resource = new KubernetesServiceResourceDefinition(resultRowEntry[0]?.ToString(), resultRowEntry[1]?.ToString(), resultRowEntry[3]?.ToString());
+            return resource;
+        }
+    }
+}

src/Promitor.Agents.Scraper/Validation/Factories/MetricValidatorFactory.cs

@@ -41,6 +41,8 @@ internal static IMetricValidator GetValidatorFor(ResourceType resourceType)
                     return new IoTHubMetricValidator();
                 case ResourceType.KeyVault:
                     return new KeyVaultMetricValidator();
+                case ResourceType.KubernetesService:
+                    return new KubernetesServiceMetricValidator();
                 case ResourceType.LogicApp:
                     return new LogicAppMetricValidator();
                 case ResourceType.NetworkGateway:

src/Promitor.Agents.Scraper/Validation/MetricDefinitions/ResourceTypes/KubernetesServiceMetricValidator.cs

@@ -0,0 +1,25 @@
+using System.Collections.Generic;
+using System.Linq;
+using GuardNet;
+using Promitor.Core.Scraping.Configuration.Model.Metrics;
+using Promitor.Agents.Scraper.Validation.MetricDefinitions.Interfaces;
+using Promitor.Core.Contracts.ResourceTypes;
+
+namespace Promitor.Agents.Scraper.Validation.MetricDefinitions.ResourceTypes
+{
+    internal class KubernetesServiceMetricValidator : IMetricValidator
+    {
+        public IEnumerable<string> Validate(MetricDefinition metricDefinition)
+        {
+            Guard.NotNull(metricDefinition, nameof(metricDefinition));
+
+            foreach (var resourceDefinition in metricDefinition.Resources.Cast<KubernetesServiceResourceDefinition>())
+            {
+                if (string.IsNullOrWhiteSpace(resourceDefinition.ClusterName))
+                {
+                    yield return "No Azure Kubernetes Service cluster name is configured";
+                }
+            }
+        }
+    }
+}

src/Promitor.Core.Contracts/ResourceType.cs

@@ -31,6 +31,7 @@ public enum ResourceType
         EventHubs = 26,
         ExpressRouteCircuit = 27,
         ApplicationGateway = 28,
-        NetworkGateway = 29
+        NetworkGateway = 29,
+        KubernetesService = 30
     }
 }

src/Promitor.Core.Contracts/ResourceTypes/KubernetesServiceResourceDefinition.cs

@@ -0,0 +1,13 @@
+namespace Promitor.Core.Contracts.ResourceTypes
+{
+    public class KubernetesServiceResourceDefinition : AzureResourceDefinition
+    {
+        public KubernetesServiceResourceDefinition(string subscriptionId, string resourceGroupName, string clusterName)
+            : base(ResourceType.KubernetesService, subscriptionId, resourceGroupName, clusterName)
+        {
+            ClusterName = clusterName;
+        }
+
+        public string ClusterName { get; }
+    }
+}

src/Promitor.Core.Scraping/Configuration/Serialization/v1/Core/AzureResourceDeserializerFactory.cs

@@ -66,6 +66,9 @@ public IDeserializer<AzureResourceDefinitionV1> GetDeserializerFor(ResourceType
                 case ResourceType.KeyVault:
                     var keyVaultLogger = _loggerFactory.CreateLogger<KeyVaultDeserializer>();
                     return new KeyVaultDeserializer(keyVaultLogger);
+                case ResourceType.KubernetesService:
+                    var KubernetesServiceLogger = _loggerFactory.CreateLogger<KubernetesServiceDeserializer>();
+                    return new KubernetesServiceDeserializer(KubernetesServiceLogger);
                 case ResourceType.LogicApp:
                     var logicAppLogger = _loggerFactory.CreateLogger<LogicAppDeserializer>();
                     return new LogicAppDeserializer(logicAppLogger);

src/Promitor.Core.Scraping/Configuration/Serialization/v1/Mapping/V1MappingProfile.cs

@@ -38,6 +38,7 @@ public V1MappingProfile()
             CreateMap<GenericResourceV1, GenericAzureResourceDefinition>();
             CreateMap<IoTHubResourceV1, IoTHubResourceDefinition>();
             CreateMap<KeyVaultResourceV1, KeyVaultResourceDefinition>();
+            CreateMap<KubernetesServiceResourceV1, KubernetesServiceResourceDefinition>();
             CreateMap<LogicAppResourceV1, LogicAppResourceDefinition>();
             CreateMap<NetworkGatewayResourceV1, NetworkGatewayResourceDefinition>();
             CreateMap<NetworkInterfaceResourceV1, NetworkInterfaceResourceDefinition>();
@@ -74,6 +75,7 @@ public V1MappingProfile()
                 .Include<GenericResourceV1, GenericAzureResourceDefinition>()
                 .Include<IoTHubResourceV1, IoTHubResourceDefinition>()
                 .Include<KeyVaultResourceV1, KeyVaultResourceDefinition>()
+                .Include<KubernetesServiceResourceV1, KubernetesServiceResourceDefinition>()
                 .Include<LogicAppResourceV1, LogicAppResourceDefinition>()
                 .Include<NetworkGatewayResourceV1, NetworkGatewayResourceDefinition>()
                 .Include<NetworkInterfaceResourceV1, NetworkInterfaceResourceDefinition>()

src/Promitor.Core.Scraping/Configuration/Serialization/v1/Model/ResourceTypes/KubernetesServiceResourceV1.cs

@@ -0,0 +1,13 @@
+namespace Promitor.Core.Scraping.Configuration.Serialization.v1.Model.ResourceTypes
+{
+    /// <summary>
+    /// Contains the configuration required to scrape an Azure Kubernetes Service.
+    /// </summary>
+    public class KubernetesServiceResourceV1 : AzureResourceDefinitionV1
+    {
+        /// <summary>
+        /// The name of the Azure Kubernetes Service to get metrics for.
+        /// </summary>
+        public string ClusterName { get; set; }
+    }
+}

src/Promitor.Core.Scraping/Configuration/Serialization/v1/Providers/KubernetesServiceDeserializer.cs

@@ -0,0 +1,14 @@
+using Microsoft.Extensions.Logging;
+using Promitor.Core.Scraping.Configuration.Serialization.v1.Model.ResourceTypes;
+
+namespace Promitor.Core.Scraping.Configuration.Serialization.v1.Providers
+{
+    public class KubernetesServiceDeserializer : ResourceDeserializer<KubernetesServiceResourceV1>
+    {
+        public KubernetesServiceDeserializer(ILogger<KubernetesServiceDeserializer> logger) : base(logger)
+        {
+            Map(resource => resource.ClusterName)
+                .IsRequired();
+        }
+    }
+}

src/Promitor.Core.Scraping/Factories/MetricScraperFactory.cs

@@ -62,6 +62,8 @@ public IScraper<IAzureResourceDefinition> CreateScraper(ResourceType metricDefin
                     return new IoTHubScraper(scraperConfiguration);
                 case ResourceType.KeyVault:
                     return new KeyVaultScraper(scraperConfiguration);
+                case ResourceType.KubernetesService:
+                    return new KubernetesServiceScraper(scraperConfiguration);
                 case ResourceType.NetworkGateway:
                     return new NetworkGatewayScraper(scraperConfiguration);
                 case ResourceType.NetworkInterface:
@@ -91,7 +93,7 @@ public IScraper<IAzureResourceDefinition> CreateScraper(ResourceType metricDefin
                 case ResourceType.WebApp:
                     return new WebAppScraper(scraperConfiguration);
                 default:
-                    throw new ArgumentOutOfRangeException();
+                    throw new ArgumentOutOfRangeException("ResourceType", metricDefinitionResourceType, "Matching scraper not found");
             }
         }
     }

src/Promitor.Core.Scraping/ResourceTypes/KubernetesServiceScraper.cs

@@ -0,0 +1,22 @@
+using Promitor.Core.Contracts;
+using Promitor.Core.Contracts.ResourceTypes;
+using Promitor.Core.Scraping.Configuration.Model.Metrics;
+
+namespace Promitor.Core.Scraping.ResourceTypes
+{
+    public class KubernetesServiceScraper : AzureMonitorScraper<KubernetesServiceResourceDefinition>
+    {
+        private const string ResourceUriTemplate = "subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.ContainerService/managedClusters/{2}";
+
+        public KubernetesServiceScraper(ScraperConfiguration scraperConfiguration) :
+            base(scraperConfiguration)
+        {
+        }
+
+        /// <inheritdoc />
+        protected override string BuildResourceUri(string subscriptionId, ScrapeDefinition<IAzureResourceDefinition> scrapeDefinition, KubernetesServiceResourceDefinition resource)
+        {
+            return string.Format(ResourceUriTemplate, subscriptionId, scrapeDefinition.ResourceGroupName, resource.ClusterName);
+        }
+    }
+}

src/Promitor.Tests.Unit/Builders/Metrics/v1/MetricsDeclarationBuilder.cs

@@ -608,6 +608,23 @@ public MetricsDeclarationBuilder WithKeyVaultMetric(string metricName = "promito
             return this;
         }
 
+        public MetricsDeclarationBuilder WithKubernetesServiceMetric(string metricName = "promitor-aks",
+            string metricDescription = "Description for a metric",
+            string clusterName = "promitor-aks",
+            string azureMetricName = "kube_node_status_condition",
+            string resourceDiscoveryGroupName = "",
+            bool omitResource = false)
+        {
+            var resource = new KubernetesServiceResourceV1
+            {
+                ClusterName = clusterName
+            };
+
+            CreateAndAddMetricDefinition(ResourceType.KubernetesService, metricName, metricDescription, resourceDiscoveryGroupName, omitResource, azureMetricName, resource);
+
+            return this;
+        }
+
         private void CreateAndAddMetricDefinition(ResourceType resourceType, string metricName, string metricDescription, string resourceDiscoveryGroupName, bool omitResource, string azureMetricName, AzureResourceDefinitionV1 resource, string metricDimension = null)
         {
             CreateAndAddMetricDefinition(resourceType, metricName, metricDescription, resourceDiscoveryGroupName, omitResource, azureMetricName, new List<AzureResourceDefinitionV1> {resource}, metricDimension);

src/Promitor.Tests.Unit/Serialization/v1/Providers/KubernetesServiceDeserializerTests.cs

@@ -0,0 +1,45 @@
+using Promitor.Core.Scraping.Configuration.Serialization;
+using Promitor.Core.Scraping.Configuration.Serialization.v1.Model;
+using Promitor.Core.Scraping.Configuration.Serialization.v1.Model.ResourceTypes;
+using Promitor.Core.Scraping.Configuration.Serialization.v1.Providers;
+using System.ComponentModel;
+using Xunit;
+
+namespace Promitor.Tests.Unit.Serialization.v1.Providers
+{
+    [Category("Unit")]
+    public class KubernetesServiceDeserializerTests : ResourceDeserializerTest<KubernetesServiceDeserializer>
+    {
+        private readonly KubernetesServiceDeserializer _deserializer;
+
+        public KubernetesServiceDeserializerTests()
+        {
+            _deserializer = new KubernetesServiceDeserializer(Logger);
+        }
+
+        [Fact]
+        public void Deserialize_ClusterNameSupplied_SetsClusterName()
+        {
+            const string clusterName = "promitor-aks";
+            YamlAssert.PropertySet<KubernetesServiceResourceV1, AzureResourceDefinitionV1, string>(
+                _deserializer,
+                $"clusterName: {clusterName}",
+                clusterName,
+                r => r.ClusterName);
+        }
+
+        [Fact]
+        public void Deserialize_ClusterNameNotSupplied_Null()
+        {
+            YamlAssert.PropertyNull<KubernetesServiceResourceV1, AzureResourceDefinitionV1>(
+                _deserializer,
+                "resourceGroupName: promitor-group",
+                r => r.ClusterName);
+        }
+
+        protected override IDeserializer<AzureResourceDefinitionV1> CreateDeserializer()
+        {
+            return new KubernetesServiceDeserializer(Logger);
+        }
+    }
+}