Provide support for Atlassian Statuspage as sink (#1154)

Co-authored-by: Adam Connelly <[email protected]>
tomkerkhove · Jul 16, 2020 · 40f533d · 40f533d
1 parent 1af35a5
commit 40f533d
Show file tree

Hide file tree

Showing 37 changed files with 1,237 additions and 302 deletions.
diff --git a/.gitignore b/.gitignore
@@ -301,6 +301,8 @@ docs/_site/*
 #MAC
 .DS_Store
 
+# Custom
 *.orig
 changelog/public/
 changelog/resources/_gen/
+src/docker-compose.vs.debug.yml
diff --git a/changelog/content/experimental/unreleased.md b/changelog/content/experimental/unreleased.md
@@ -14,3 +14,8 @@ version:
 - {{% tag added %}} Provide suggestions when unknown fields are found in the metrics config. [#1105](https://github.com/tomkerkhove/promitor/issues/1105).
 - {{% tag added %}} Add validation to ensure the scraping schedule is a valid Cron expression. [#1103](https://github.com/tomkerkhove/promitor/issues/1103).
 - {{% tag changed %}} Handle validation failures on startup more gracefully. [#1113](https://github.com/tomkerkhove/promitor/issues/1113).
+- {{% tag added %}} Provide support for pushing metrics to Atlassian Statuspage
+ ([docs](https://promitor.io/configuration/v2.x/runtime#atlassian-statuspage) | [#1152](https://github.com/tomkerkhove/promitor/issues/1152))
+- {{% tag added %}} Provide suggestions when unknown fields are found in the metrics config. [#1105](https://github.com/tomkerkhove/promitor/issues/1105).
+- {{% tag added %}} New validation rule to ensure the scraping schedule is a valid Cron expression. [#1103](https://github.com/tomkerkhove/promitor/issues/1103).
+- {{% tag added %}} New validation rule to ensure declarative or dynamic discovery for metrics to scrape are configured
diff --git a/config/promitor/scraper/metrics.yaml b/config/promitor/scraper/metrics.yaml
@@ -10,132 +10,6 @@ metricDefaults:
     # Every minute
     schedule: "0 * * ? * *"
 metrics:
-  - name: promitor_demo_container_cpu
-    description: "The CPU of our containers in a container group."
-    resourceType: ContainerInstance
-    azureMetricConfiguration:
-      metricName: CpuUsage
-      dimension:
-        name: containerName
-      aggregation:
-        type: Average
-    resources:
-    - containerGroup: promitor-container-group
-      resourceGroupName: promitor-sources
-  - name: promitor_demo_generic_queue_size
-    description: "Amount of active messages of the 'orders' queue (determined with Generic provider)"
-    resourceType: Generic
-    labels:
-      app: promitor
-    azureMetricConfiguration:
-      metricName: ActiveMessages
-      aggregation:
-        type: Average
-    resources:
-    - resourceUri: Microsoft.ServiceBus/namespaces/promitor-messaging
-      filter: EntityName eq 'orders'
-  - name: promitor_demo_generic_namespace_size
-    description: "Size of all queues in our Azure Service Bus namespace (determined with Generic provider)"
-    resourceType: Generic
-    scraping:
-      # Every 2 minutes
-      schedule: "0 */2 * ? * *"
-    azureMetricConfiguration:
-      metricName: ActiveMessages
-      aggregation:
-        type: Average
-    resources:
-      # filter is deliberately omitted given filter is optional
-    - resourceUri: Microsoft.ServiceBus/namespaces/promitor-messaging
-  - name: promitor_demo_servicebusqueue_queue_size_discovered
-    description: "Amount of active messages of the Service Bus queues"
-    resourceType: ServiceBusQueue
-    azureMetricConfiguration:
-      metricName: ActiveMessages
-      aggregation:
-        type: Average
-        # Optionally override the default aggregation interval (metricDefaults.aggregation.interval)
-        interval: 00:15:00
-    resourceDiscoveryGroups:
-    - name: service-bus-landscape
-  - name: promitor_demo_servicebusqueue_queue_size
-    description: "Amount of active messages of the 'orders' queue (determined with ServiceBusQueue provider)"
-    resourceType: ServiceBusQueue
-    azureMetricConfiguration:
-      metricName: ActiveMessages
-      aggregation:
-        type: Average
-        # Optionally override the default aggregation interval (metricDefaults.aggregation.interval)
-        interval: 00:15:00
-    resources:
-    - namespace: promitor-messaging
-      queueName: orders
-    - namespace: promitor-messaging
-      queueName: sales
-    - namespace: promitor-messaging-other-subscription
-      queueName: orders
-      subscriptionId: 0329dd2a-59dc-4493-aa54-cb01cb027dc2
-      resourceGroupName: promitor-sources
-  - name: promitor_demo_azurestoragequeue_queue_size
-    description: "Approximate amount of messages in 'orders' queue (determined with StorageQueue provider)"
-    resourceType: StorageQueue
-    scraping:
-      # Every 2 minutes
-      schedule: "0 */2 * ? * *"
-    azureMetricConfiguration:
-      metricName: MessageCount
-      aggregation:
-        type: Total
-    resources:
-    - accountName: promitor
-      queueName: orders
-      sasToken:
-        rawValue: "?sv=2018-03-28&ss=bfqt&srt=sco&sp=rwla&se=2022-08-07T00:16:01Z&st=2019-08-06T16:16:01Z&spr=https&sig=Ik4jprS89kGIFRM0qaQpXrv0ttP3pnlhmGQuYVQ7cbA%3D"
-  - name: promitor_demo_azurestoragequeue_queue_timespentinqueue
-    description: "Approximate amount of time that the oldest message has been in 'orders' queue (determined with StorageQueue provider)"
-    resourceType: StorageQueue
-    azureMetricConfiguration:
-      metricName: TimeSpentInQueue
-      aggregation:
-        type: Total
-    resources:
-    - accountName: promitor
-      queueName: orders
-      sasToken:
-        environmentVariable: SECRETS_STORAGEQUEUE_SAS
-  - name: promitor_demo_azuresqldb_dtu
-    description: "The DTU consumption percentage used by an Azure SQL Database."
-    resourceType: SqlDatabase
-    azureMetricConfiguration:
-      metricName: dtu_consumption_percent
-      aggregation:
-        type: Average
-    resources:
-    - serverName: promitor
-      databaseName: promitor-db-1
-  - name: promitor_demo_webapp_cpu
-    description: "Amount of CPU time used for an Azure Web App"
-    resourceType: WebApp
-    azureMetricConfiguration:
-      metricName: CpuTime
-      aggregation:
-        type: Total
-    resources:
-    - webAppName: promitor-web-app
-      resourceGroupName: promitor-sources
-    - webAppName: promitor-web-app
-      resourceGroupName: promitor-sources
-      slotName: staging
-  - name: promitor_demo_function_requests
-    description: "Amount of requests for an Azure Function App"
-    resourceType: FunctionApp
-    azureMetricConfiguration:
-      metricName: Requests
-      aggregation:
-        type: Total
-    resources:
-    - functionAppName: promitor-function-app
-      resourceGroupName: promitor-sources
   - name: promitor_demo_appplan_percentage_cpu
     description: "Average percentage of memory usage on an Azure App Plan"
     resourceType: AppPlan
@@ -199,4 +73,4 @@ metrics:
       aggregation:
         type: Total
     resourceDiscoveryGroups:
-    - name: logic-apps-unfiltered
+    - name: logic-apps-unfiltered
diff --git a/config/promitor/scraper/runtime.yaml b/config/promitor/scraper/runtime.yaml
@@ -9,6 +9,11 @@ metricSinks:
     host: graphite
     port: 8125
     metricPrefix: promitor.
+  atlassianStatuspage:
+    pageId: y79z9b78ybgs
+    systemMetricMapping:
+    - id: nfkgnrwpn545
+      promitorMetricName: promitor_demo_appplan_percentage_cpu
 metricsConfiguration:
   absolutePath: /config/metrics-declaration.yaml
 telemetry:

diff --git a/docs/configuration/v2.x/runtime.md b/docs/configuration/v2.x/runtime.md
@@ -66,9 +66,39 @@ by providing the metric information to the configured sinks.
 
 As of today, we support the follow sinks:
 
+- **Atlassian Statuspage**
 - **Prometheus Scraping Endpoint**
 - **StatsD**
 
+### Atlassian Statuspage
+
+![Availability Badge](https://img.shields.io/badge/Available%20Starting-v2.0-green.svg)
+
+In order to expose a Prometheus Scraping endpoint, you'll need to configure the sink:
+
+- `atlassianStatuspage.pageId` - Defines the id of the Atlassian Statuspage to report to.
+- `atlassianStatuspage.systemMetricMapping` - Defines a mapping of the scraped metric by Promitor and to which
+ Atlassian Statuspage system metric it should be reported to. Here's what we expect:
+  - `id` - Id of the Atlassian Statuspage system metric
+  - `promitorMetricName` - Name of the Promitor metric which needs to be reported
+
+Next to that, `PROMITOR_ATLASSIAN_STATUSPAGE_APIKEY` environment variable is required which contains the API Key
+ for Atlassian Statuspage.
+
+```yaml
+metricSinks:
+  atlassianStatuspage:
+    pageId: XXX          # Mandatory
+    systemMetricMapping: # Mandatory to have at least one mapping
+    - id: ABC
+      promitorMetricName: promitor_demo_appplan_percentage_cpu
+```
+
+> :warning: **As of today, metric labels, resource discovery and multi-resource scraping are not supported.**
+>  
+> This is because Promitor will report the different resource metrics to the same Atlassian metric which will mix metrics
+> which becomes confusing.
+
 ### Prometheus Scraping Endpoint
 
 ![Availability Badge](https://img.shields.io/badge/Available%20Starting-v1.6-green.svg)
@@ -110,6 +140,10 @@ metricSinks:
     metricPrefix: promitor.
 ```
 
+> :warning: **As of today, metric labels are not supported.**
+>  
+> Unfortunately, this is not supported in the specifiaction.
+
 ## Metric Configuration
 
 Promitor will scrape the Azure Monitor metrics that are configured via a metric

diff --git a/docs/index.md b/docs/index.md
@@ -56,6 +56,7 @@ and vote for features!
   - [What labels do we provide?](metrics/labels)
 - **Configuration**
   - [Overview of metric sinks](configuration/v1.x/runtime#metric-sinks)
+    - [Atlassian Statuspage](configuration/v2.x/runtime#atlassian-statuspage)
     - [Prometheus Scraping Endpoint](configuration/v1.x/runtime#prometheus-scraping-endpoint)
     - [StatsD](configuration/v1.x/runtime#statsd)
   - [Authentication with Azure Monitor](configuration/v1.x/azure-monitor)

diff --git a/src/Promitor.Agents.Scraper/Configuration/Sinks/MetricSinkConfiguration.cs b/src/Promitor.Agents.Scraper/Configuration/Sinks/MetricSinkConfiguration.cs
@@ -1,4 +1,5 @@
-using Promitor.Integrations.Sinks.Prometheus.Configuration;
+using Promitor.Integrations.Sinks.Atlassian.Statuspage.Configuration;
+using Promitor.Integrations.Sinks.Prometheus.Configuration;
 using Promitor.Integrations.Sinks.Statsd.Configuration;
 
 namespace Promitor.Agents.Scraper.Configuration.Sinks
@@ -7,5 +8,6 @@ public class MetricSinkConfiguration
     {
         public StatsdSinkConfiguration Statsd { get; set; }
         public PrometheusScrapingEndpointSinkConfiguration PrometheusScrapingEndpoint { get; set; }
+        public AtlassianStatusPageSinkConfiguration AtlassianStatuspage { get; set; }
     }
 }
diff --git a/src/Promitor.Agents.Scraper/Discovery/ResourceDiscoveryClient.cs b/src/Promitor.Agents.Scraper/Discovery/ResourceDiscoveryClient.cs
@@ -20,18 +20,18 @@ public class ResourceDiscoveryClient
         private readonly JsonSerializerSettings _serializerSettings = new JsonSerializerSettings { TypeNameHandling = TypeNameHandling.Objects };
         private readonly IOptionsMonitor<ResourceDiscoveryConfiguration> _configuration;
         private readonly ILogger<ResourceDiscoveryClient> _logger;
-        private readonly IHttpClientFactory _httpClientFactory;
+        private readonly HttpClient _httpClient;
 
-        public ResourceDiscoveryClient(IHttpClientFactory httpClientFactory, IOptionsMonitor<ResourceDiscoveryConfiguration> configuration, ILogger<ResourceDiscoveryClient> logger)
+        public ResourceDiscoveryClient(HttpClient httpClient, IOptionsMonitor<ResourceDiscoveryConfiguration> configuration, ILogger<ResourceDiscoveryClient> logger)
         {
-            Guard.NotNull(httpClientFactory, nameof(httpClientFactory));
+            Guard.NotNull(httpClient, nameof(httpClient));
             Guard.NotNull(configuration, nameof(configuration));
             Guard.NotNull(logger, nameof(logger));
             Guard.For<Exception>(() => configuration.CurrentValue.IsConfigured == false, "Resource Discovery is not configured");
 
             _logger = logger;
+            _httpClient = httpClient;
             _configuration = configuration;
-            _httpClientFactory = httpClientFactory;
         }
 
         public async Task<List<AzureResourceDefinition>> GetAsync(string resourceDiscoveryGroupName)
@@ -63,13 +63,13 @@ private async Task<string> SendGetRequestAsync(string uri)
 
         private async Task<HttpResponseMessage> SendRequestToApiAsync(HttpRequestMessage request)
         {
-            var client = CreateHttpClient();
             using (var dependencyMeasurement = DependencyMeasurement.Start())
             {
                 HttpResponseMessage response = null;
                 try
                 {
-                    response = await client.SendAsync(request);
+                    _httpClient.BaseAddress = new Uri($"http://{_configuration.CurrentValue.Host}:{_configuration.CurrentValue.Port}");
+                    response = await _httpClient.SendAsync(request);
                     _logger.LogRequest(request, response, dependencyMeasurement.Elapsed);
 
                     return response;
@@ -81,12 +81,5 @@ private async Task<HttpResponseMessage> SendRequestToApiAsync(HttpRequestMessage
                 }
             }
         }
-
-        private HttpClient CreateHttpClient()
-        {
-            var httpClient = _httpClientFactory.CreateClient("Promitor Resource Discovery");
-            httpClient.BaseAddress = new Uri($"http://{_configuration.CurrentValue.Host}:{_configuration.CurrentValue.Port}");
-            return httpClient;
-        }
     }
 }
diff --git a/src/Promitor.Agents.Scraper/Dockerfile b/src/Promitor.Agents.Scraper/Dockerfile
@@ -8,6 +8,7 @@ COPY Promitor.Core.Telemetry/* Promitor.Core.Telemetry/
 COPY Promitor.Integrations.AzureMonitor/* Promitor.Integrations.AzureMonitor/
 COPY Promitor.Integrations.AzureStorage/* Promitor.Integrations.AzureStorage/
 COPY Promitor.Integrations.Sinks.Statsd/* Promitor.Integrations.Sinks.Statsd/
+COPY Promitor.Integrations.Sinks.Atlassian.Statuspage/* Promitor.Integrations.Sinks.Atlassian.Statuspage/
 COPY Promitor.Integrations.Sinks.Prometheus/* Promitor.Integrations.Sinks.Prometheus/
 COPY Promitor.Agents.Scraper/* Promitor.Agents.Scraper/
 RUN dotnet publish Promitor.Agents.Scraper/Promitor.Agents.Scraper.csproj --configuration release --output app

diff --git a/src/Promitor.Agents.Scraper/Extensions/IServiceCollectionExtensions.cs b/src/Promitor.Agents.Scraper/Extensions/IServiceCollectionExtensions.cs
@@ -23,6 +23,8 @@
 using Promitor.Core.Metrics.Sinks;
 using Promitor.Core.Scraping.Configuration.Runtime;
 using Promitor.Integrations.AzureMonitor.Configuration;
+using Promitor.Integrations.Sinks.Atlassian.Statuspage;
+using Promitor.Integrations.Sinks.Atlassian.Statuspage.Configuration;
 using Promitor.Integrations.Sinks.Prometheus;
 using Promitor.Integrations.Sinks.Prometheus.Configuration;
 using Promitor.Integrations.Sinks.Statsd;
@@ -40,7 +42,6 @@ public static class IServiceCollectionExtensions
         /// <param name="services">Collections of services in application</param>
         public static IServiceCollection DefineDependencies(this IServiceCollection services)
         {
-            services.AddTransient<ResourceDiscoveryClient>();
             services.AddTransient<ResourceDiscoveryRepository>();
             services.AddTransient<IMetricsDeclarationProvider, MetricsDeclarationProvider>();
             services.AddTransient<IRuntimeMetricsCollector, RuntimeMetricsCollector>();
@@ -98,6 +99,11 @@ public static IServiceCollection UseMetricSinks(this IServiceCollection services
                 AddPrometheusMetricSink(services);
             }
 
+            if (metricSinkConfiguration?.AtlassianStatuspage != null)
+            {
+                AddAtlassianStatuspageMetricSink(services);
+            }
+
             services.TryAddSingleton<MetricSinkWriter>();
 
             return services;
@@ -108,6 +114,11 @@ private static void AddPrometheusMetricSink(IServiceCollection services)
             services.AddTransient<IMetricSink, PrometheusScrapingEndpointMetricSink>();
         }
 
+        private static void AddAtlassianStatuspageMetricSink(IServiceCollection services)
+        {
+            services.AddTransient<IMetricSink, AtlassianStatuspageMetricSink>();
+        }
+
         private static void AddStatsdMetricSink(IServiceCollection services, StatsdSinkConfiguration statsdConfiguration)
         {
             services.AddTransient<IMetricSink, StatsdMetricSink>();
@@ -143,7 +154,9 @@ public static IServiceCollection ConfigureYamlConfiguration(this IServiceCollect
             services.Configure<ResourceDiscoveryConfiguration>(configuration.GetSection("resourceDiscovery"));
             services.Configure<TelemetryConfiguration>(configuration.GetSection("telemetry"));
             services.Configure<ServerConfiguration>(configuration.GetSection("server"));
-            services.Configure<PrometheusScrapingEndpointSinkConfiguration>(configuration.GetSection("prometheus"));
+            services.Configure<PrometheusScrapingEndpointSinkConfiguration>(configuration.GetSection("metricSinks:prometheus"));
+            services.Configure<StatsdSinkConfiguration>(configuration.GetSection("metricSinks:statsd"));
+            services.Configure<AtlassianStatusPageSinkConfiguration>(configuration.GetSection("metricSinks:atlassianStatuspage"));
             services.Configure<ApplicationInsightsConfiguration>(configuration.GetSection("telemetry:applicationInsights"));
             services.Configure<ContainerLogConfiguration>(configuration.GetSection("telemetry:containerLogs"));
             services.Configure<ScrapeEndpointConfiguration>(configuration.GetSection("prometheus:scrapeEndpoint"));

diff --git a/src/Promitor.Agents.Scraper/Promitor.Agents.Scraper.csproj b/src/Promitor.Agents.Scraper/Promitor.Agents.Scraper.csproj
@@ -53,6 +53,7 @@
     <ProjectReference Include="..\Promitor.Core\Promitor.Core.csproj" />
     <ProjectReference Include="..\Promitor.Integrations.AzureMonitor\Promitor.Integrations.AzureMonitor.csproj" />
     <ProjectReference Include="..\Promitor.Integrations.AzureStorage\Promitor.Integrations.AzureStorage.csproj" />
+    <ProjectReference Include="..\Promitor.Integrations.Sinks.Atlassian.Statuspage\Promitor.Integrations.Sinks.Atlassian.Statuspage.csproj" />
     <ProjectReference Include="..\Promitor.Integrations.Sinks.Prometheus\Promitor.Integrations.Sinks.Prometheus.csproj" />
     <ProjectReference Include="..\Promitor.Integrations.Sinks.Statsd\Promitor.Integrations.Sinks.Statsd.csproj" />
   </ItemGroup>