Provide support for Atlassian Statuspage as sink

.gitignore

@@ -301,6 +301,8 @@ docs/_site/*
 #MAC
 .DS_Store
 
+# Custom
 *.orig
 changelog/public/
 changelog/resources/_gen/
+src/docker-compose.vs.debug.yml

changelog/content/experimental/unreleased.md

@@ -8,9 +8,14 @@ version:
 - {{% tag removed %}} Support for Prometheus legacy configuration ([deprecation notice](https://changelog.promitor.io/#prometheus-legacy-configuration))
 - {{% tag removed %}} Support for Swagger UI 2.0 ([deprecation notice](https://changelog.promitor.io/#swagger-ui-2-0))
 - {{% tag removed %}} Support for Swagger 2.0 ([deprecation notice](https://changelog.promitor.io/#swagger-2-0))
+- {{% tag added %}} Support for scraping Azure Logic Apps ([docs](https://promitor.io/configuration/v2.x/metrics/logic-apps)
+ | [#372](https://github.com/tomkerkhove/promitor/issues/314))
+- {{% tag added %}} New validation rule to ensure at least one resource or resource collection is configured to scrape
+- {{% tag added %}} Provide suggestions when unknown fields are found in the metrics config. [#1105](https://github.com/tomkerkhove/promitor/issues/1105).
+- {{% tag added %}} Add validation to ensure the scraping schedule is a valid Cron expression. [#1103](https://github.com/tomkerkhove/promitor/issues/1103).
 - {{% tag changed %}} Handle validation failures on startup more gracefully. [#1113](https://github.com/tomkerkhove/promitor/issues/1113).
 - {{% tag added %}} Provide support for pushing metrics to Atlassian Statuspage
- ([docs](https://promitor.io/configuration/v2.x/runtime#atlassian-statuspage) | [#933](https://github.com/tomkerkhove/promitor/issues/1152))
+ ([docs](https://promitor.io/configuration/v2.x/runtime#atlassian-statuspage) | [#1152](https://github.com/tomkerkhove/promitor/issues/1152))
 - {{% tag added %}} Provide suggestions when unknown fields are found in the metrics config. [#1105](https://github.com/tomkerkhove/promitor/issues/1105).
 - {{% tag added %}} New validation rule to ensure the scraping schedule is a valid Cron expression. [#1103](https://github.com/tomkerkhove/promitor/issues/1103).
 - {{% tag added %}} New validation rule to ensure declarative or dynamic discovery for metrics to scrape are configured

config/promitor/scraper/metrics.yaml

@@ -19,4 +19,58 @@ metrics:
         type: Average
     resources:
     - appPlanName: promitor-app-plan
-      resourceGroupName: promitor-sources
+      resourceGroupName: promitor-sources
+  - name: azure_container_registry_total_pull_count_discovered
+    description: "Amount of images that were pulled from the container registry"
+    resourceType: ContainerRegistry
+    azureMetricConfiguration:
+      metricName: TotalPullCount
+      aggregation:
+        type: Average
+    resourceDiscoveryGroups:
+    - name: container-registry-landscape
+  - name: promitor_demo_appplan_percentage_cpu_discovered
+    description: "Average percentage of memory usage on an Azure App Plan"
+    resourceType: AppPlan
+    azureMetricConfiguration:
+      metricName: MemoryPercentage
+      aggregation:
+        type: Average
+    resourceDiscoveryGroups:
+    - name: app-plan-landscape
+  - name: promitor_demo_webapp_cpu_discovery
+    description: "Amount of CPU time used for an Azure Web App"
+    resourceType: WebApp
+    azureMetricConfiguration:
+      metricName: CpuTime
+      aggregation:
+        type: Total
+    resourceDiscoveryGroups:
+    - name: web-app-landscape
+  - name: promitor_demo_function_memory_discovery
+    description: "Average memory for an Azure Function App"
+    resourceType: FunctionApp
+    azureMetricConfiguration:
+      metricName: MemoryWorkingSet
+      aggregation:
+        type: Average
+    resourceDiscoveryGroups:
+    - name: function-app-landscape
+  - name: azure_logic_apps_failed_run
+    description: "Total amount of failed runs for Azure Logic Apps"
+    resourceType: LogicApp
+    azureMetricConfiguration:
+      metricName: RunsFailed
+      aggregation:
+        type: Total
+    resources:
+    - workflowName: promitor-automation-github-ci-scraper
+  - name: azure_logic_apps_failed_run_discovery
+    description: "Total amount of failed runs for Azure Logic Apps"
+    resourceType: LogicApp
+    azureMetricConfiguration:
+      metricName: RunsFailed
+      aggregation:
+        type: Total
+    resourceDiscoveryGroups:
+    - name: logic-apps-unfiltered

docs/configuration/v2.x/runtime.md

@@ -94,7 +94,10 @@ metricSinks:
       promitorMetricName: promitor_demo_appplan_percentage_cpu
 ```
 
-> :warning: Metric labels are not supported.
+> :warning: **As of today, metric labels, resource discovery and multi-resource scraping are not supported.**
+>  
+> This is because Promitor will report the different resource metrics to the same Atlassian metric which will mix metrics
+> which becomes confusing.
 
 ### Prometheus Scraping Endpoint
 
@@ -137,7 +140,9 @@ metricSinks:
     metricPrefix: promitor.
 ```
 
-> :warning: Metric labels are not supported.
+> :warning: **As of today, metric labels are not supported.**
+>  
+> Unfortunately, this is not supported in the specifiaction.
 
 ## Metric Configuration
 

src/Promitor.Agents.Scraper/Discovery/ResourceDiscoveryClient.cs

@@ -11,7 +11,6 @@
 using Newtonsoft.Json;
 using Promitor.Agents.Core.Serialization;
 using Promitor.Agents.Scraper.Configuration;
-using Promitor.Core;
 using Promitor.Core.Contracts;
 
 namespace Promitor.Agents.Scraper.Discovery
@@ -21,18 +20,18 @@ public class ResourceDiscoveryClient
         private readonly JsonSerializerSettings _serializerSettings = new JsonSerializerSettings { TypeNameHandling = TypeNameHandling.Objects };
         private readonly IOptionsMonitor<ResourceDiscoveryConfiguration> _configuration;
         private readonly ILogger<ResourceDiscoveryClient> _logger;
-        private readonly IHttpClientFactory _httpClientFactory;
+        private readonly HttpClient _httpClient;
 
-        public ResourceDiscoveryClient(IHttpClientFactory httpClientFactory, IOptionsMonitor<ResourceDiscoveryConfiguration> configuration, ILogger<ResourceDiscoveryClient> logger)
+        public ResourceDiscoveryClient(HttpClient httpClient, IOptionsMonitor<ResourceDiscoveryConfiguration> configuration, ILogger<ResourceDiscoveryClient> logger)
         {
-            Guard.NotNull(httpClientFactory, nameof(httpClientFactory));
+            Guard.NotNull(httpClient, nameof(httpClient));
             Guard.NotNull(configuration, nameof(configuration));
             Guard.NotNull(logger, nameof(logger));
             Guard.For<Exception>(() => configuration.CurrentValue.IsConfigured == false, "Resource Discovery is not configured");
 
             _logger = logger;
+            _httpClient = httpClient;
             _configuration = configuration;
-            _httpClientFactory = httpClientFactory;
         }
 
         public async Task<List<AzureResourceDefinition>> GetAsync(string resourceDiscoveryGroupName)
@@ -64,13 +63,13 @@ private async Task<string> SendGetRequestAsync(string uri)
 
         private async Task<HttpResponseMessage> SendRequestToApiAsync(HttpRequestMessage request)
         {
-            var client = CreateHttpClient();
             using (var dependencyMeasurement = DependencyMeasurement.Start())
             {
                 HttpResponseMessage response = null;
                 try
                 {
-                    response = await client.SendAsync(request);
+                    _httpClient.BaseAddress = new Uri($"http://{_configuration.CurrentValue.Host}:{_configuration.CurrentValue.Port}");
+                    response = await _httpClient.SendAsync(request);
                     _logger.LogRequest(request, response, dependencyMeasurement.Elapsed);
 
                     return response;
@@ -82,12 +81,5 @@ private async Task<HttpResponseMessage> SendRequestToApiAsync(HttpRequestMessage
                 }
             }
         }
-
-        private HttpClient CreateHttpClient()
-        {
-            var httpClient = _httpClientFactory.CreateClient(Http.Clients.ResourceDiscovery);
-            httpClient.BaseAddress = new Uri($"http://{_configuration.CurrentValue.Host}:{_configuration.CurrentValue.Port}");
-            return httpClient;
-        }
     }
 }

src/Promitor.Agents.Scraper/Extensions/IServiceCollectionExtensions.cs

@@ -42,14 +42,12 @@ public static class IServiceCollectionExtensions
         /// <param name="services">Collections of services in application</param>
         public static IServiceCollection DefineDependencies(this IServiceCollection services)
         {
-            services.AddTransient<ResourceDiscoveryClient>();
             services.AddTransient<ResourceDiscoveryRepository>();
             services.AddTransient<IMetricsDeclarationProvider, MetricsDeclarationProvider>();
             services.AddTransient<IRuntimeMetricsCollector, RuntimeMetricsCollector>();
             services.AddTransient<MetricScraperFactory>();
             services.AddTransient<ConfigurationSerializer>();
             services.AddSingleton<AzureMonitorClientFactory>();
-            services.AddSingleton<AtlassianStatuspageClient>();
 
             services.AddSingleton<IDeserializer<MetricsDeclarationV1>, V1Deserializer>();
             services.AddSingleton<IDeserializer<AzureMetadataV1>, AzureMetadataDeserializer>();

src/Promitor.Agents.Scraper/Startup.cs

@@ -10,11 +10,13 @@
 using Promitor.Agents.Core;
 using Promitor.Agents.Scraper.Configuration;
 using Promitor.Agents.Scraper.Configuration.Sinks;
+using Promitor.Agents.Scraper.Discovery;
 using Promitor.Agents.Scraper.Extensions;
 using Promitor.Agents.Scraper.Health;
 using Promitor.Core;
 using Promitor.Core.Scraping.Configuration.Serialization.v1.Mapping;
 using Promitor.Integrations.AzureMonitor.Logging;
+using Promitor.Integrations.Sinks.Atlassian.Statuspage;
 using Serilog;
 
 namespace Promitor.Agents.Scraper
@@ -32,15 +34,19 @@ public Startup(IConfiguration configuration)
         public void ConfigureServices(IServiceCollection services)
         {
             string openApiDescription = BuildOpenApiDescription(Configuration);
-            services.AddHttpClient(Http.Clients.ResourceDiscovery, client =>
+            services.AddHttpClient<ResourceDiscoveryClient>(client =>
             {
                 // Provide Promitor User-Agent
                 client.DefaultRequestHeaders.UserAgent.TryParseAdd(Http.Headers.UserAgents.Scraper);
             });
-            services.AddHttpClient(Http.Clients.AtlassianStatuspage, client =>
+            services.AddHttpClient<AtlassianStatuspageClient>(client =>
             {
                 // Provide Promitor User-Agent
                 client.DefaultRequestHeaders.UserAgent.TryParseAdd(Http.Headers.UserAgents.Scraper);
+
+                // Auth all requests
+                var apiKey = Configuration[EnvironmentVariables.Integrations.AtlassianStatuspage.ApiKey];
+                client.DefaultRequestHeaders.Add("Authorization", $"OAuth {apiKey}");
             });
 
             services.UseWebApi()

src/Promitor.Agents.Scraper/Validation/Steps/Sinks/AtlassianStatuspageMetricSinkValidationStep.cs

@@ -37,7 +37,7 @@ public ValidationResult Run()
             var errorMessages = new List<string>();
             if (string.IsNullOrWhiteSpace(atlassianStatuspageConfiguration.PageId))
             {
-                errorMessages.Add("No id of Atlassian Status page is configured");
+                errorMessages.Add("No page id of Atlassian Status page is configured");
             }
 
             if (atlassianStatuspageConfiguration.SystemMetricMapping?.Any() != true)
@@ -52,27 +52,39 @@ public ValidationResult Run()
                 }
 
                 var metricsDeclaration = _metricsDeclarationProvider.Get(true);
-                // TODO: Validate for empty system metric or empty promitor metric
+
                 foreach (var systemMetric in atlassianStatuspageConfiguration.SystemMetricMapping)
                 {
                     if (string.IsNullOrWhiteSpace(systemMetric.Id))
                     {
-                        errorMessages.Add($"System metric mapping defined without specifying a system metric id (Promitor metric name: {systemMetric.PromitorMetricName}");
+                        errorMessages.Add($"System metric mapping defined without specifying a system metric id (Promitor metric name: {systemMetric.PromitorMetricName})");
                     }
                     if (string.IsNullOrWhiteSpace(systemMetric.PromitorMetricName))
                     {
                         errorMessages.Add($"System metric mapping defined without specifying a Promitor metric name (System metric id: {systemMetric.Id})");
                     }
 
-                    var isAnyDefined = metricsDeclaration.Metrics.Any(metricDefinition => metricDefinition.PrometheusMetricDefinition.Name.Equals(systemMetric.PromitorMetricName));
-                    if (isAnyDefined == false)
+                    var matchingPromitorMetric = metricsDeclaration.Metrics.FirstOrDefault(metricDefinition => metricDefinition.PrometheusMetricDefinition.Name.Equals(systemMetric.PromitorMetricName));
+                    if (matchingPromitorMetric == null)
                     {
-                        errorMessages.Add($"No configured metrics '{systemMetric.PromitorMetricName}' was found while it is mapped to page with id {systemMetric.Id}");
+                        errorMessages.Add($"Statuspage metric Id '{systemMetric.Id}' is mapped to a metric called '{systemMetric.PromitorMetricName}', but no metric was found with that name");
+                    }
+                    else
+                    {
+                        if (matchingPromitorMetric.ResourceDiscoveryGroups?.Any() == true)
+                        {
+                            errorMessages.Add("Scraping with resource discovery is not supported");
+                        }
+
+                        if (matchingPromitorMetric.Resources?.Count > 1)
+                        {
+                            errorMessages.Add("Scraping multiple resources for one metric is not supported");
+                        }
                     }
                 }
             }
 
             return errorMessages.Any() ? ValidationResult.Failure(ComponentName, errorMessages) : ValidationResult.Successful(ComponentName);
         }
     }
-}
+}

src/Promitor.Core/Http.cs

@@ -2,12 +2,6 @@
 {
     public class Http
     {
-        public class Clients
-        {
-            public const string ResourceDiscovery = "Promitor Resource Discovery";
-            public const string AtlassianStatuspage = "Atlassian Statuspage";
-        }
-
         public class Headers
         {
             public class UserAgents

src/Promitor.Docker.dcproj

@@ -10,6 +10,9 @@
     <DockerServiceName>promitor.scraper</DockerServiceName>
   </PropertyGroup>
   <ItemGroup>
+    <None Include="docker-compose.vs.debug.yml">
+      <DependentUpon>docker-compose.yml</DependentUpon>
+    </None>
     <None Include="docker-compose.override.yml">
       <DependentUpon>docker-compose.yml</DependentUpon>
     </None>

src/Promitor.Integrations.Sinks.Atlassian.Statuspage/AtlassianStatusPageClient.cs

@@ -1,13 +1,12 @@
 using System;
 using System.Net.Http;
+using System.Net.Mime;
 using System.Text;
 using System.Threading.Tasks;
 using Flurl;
 using GuardNet;
-using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
-using Promitor.Core;
 using Promitor.Integrations.Sinks.Atlassian.Statuspage.Configuration;
 
 namespace Promitor.Integrations.Sinks.Atlassian.Statuspage
@@ -19,27 +18,23 @@ public class AtlassianStatuspageClient
 
         private readonly IOptionsMonitor<AtlassianStatusPageSinkConfiguration> _sinkConfiguration;
         private readonly ILogger<AtlassianStatuspageClient> _logger;
-        private readonly IHttpClientFactory _clientFactory;
-        private readonly IConfiguration _configuration;
+        private readonly HttpClient _httpClient;
 
-        public AtlassianStatuspageClient(IHttpClientFactory clientFactory, IConfiguration configuration, IOptionsMonitor<AtlassianStatusPageSinkConfiguration> sinkConfiguration, ILogger<AtlassianStatuspageClient> logger)
+        public AtlassianStatuspageClient(HttpClient httpClient, IOptionsMonitor<AtlassianStatusPageSinkConfiguration> sinkConfiguration, ILogger<AtlassianStatuspageClient> logger)
         {
-            Guard.NotNull(clientFactory, nameof(clientFactory));
-            Guard.NotNull(configuration, nameof(configuration));
+            Guard.NotNull(httpClient, nameof(httpClient));
             Guard.NotNull(logger, nameof(logger));
             Guard.NotNull(sinkConfiguration, nameof(sinkConfiguration));
             Guard.NotNull(sinkConfiguration.CurrentValue, nameof(sinkConfiguration.CurrentValue));
 
             _sinkConfiguration = sinkConfiguration;
-            _clientFactory = clientFactory;
-            _configuration = configuration;
+            _httpClient = httpClient;
             _logger = logger;
         }
 
         public async Task ReportMetricAsync(string id, double value)
         {
             var pageId = _sinkConfiguration.CurrentValue.PageId;
-            var apiKey = _configuration[EnvironmentVariables.Integrations.AtlassianStatuspage.ApiKey];
 
             // Docs: https://developer.statuspage.io/#operation/postPagesPageIdMetricsMetricIdData
             var requestUri = ApiUrl.AppendPathSegment("pages")
@@ -51,13 +46,10 @@ public async Task ReportMetricAsync(string id, double value)
             var measurementTime = DateTimeOffset.UtcNow.ToUnixTimeSeconds();
             var request = new HttpRequestMessage(HttpMethod.Post, requestUri)
             {
-                Content = new StringContent(string.Format(MetricRequestFormat, measurementTime, value), Encoding.UTF8, "application/json")
+                Content = new StringContent(string.Format(MetricRequestFormat, measurementTime, value), Encoding.UTF8, MediaTypeNames.Application.Json)
             };
-            request.Headers.Add("Authorization", $"OAuth {apiKey}");
 
-            var client = _clientFactory.CreateClient(Http.Clients.AtlassianStatuspage);
-
-            var response = await client.SendAsync(request);
+            var response = await _httpClient.SendAsync(request);
 
             if (response.IsSuccessStatusCode == false)
             {