feat: Provide capability to define default custom labels

changelog/content/experimental/unreleased.md

@@ -8,6 +8,8 @@ version:
 
 - {{% tag added %}} Provide capability to transform metric labels in Prometheus ([docs](https://promitor.io/configuration/v2.x/runtime/scraper#prometheus-scraping-endpoint)
  | [#1596](https://github.com/tomkerkhove/promitor/issues/1596))
+- {{% tag added %}} Provide capability to define default custom labels  ([docs](https://promitor.io/configuration/v2.x/metrics/)
+ | [#1608](https://github.com/tomkerkhove/promitor/issues/1608))
 
 #### Resource Discovery
 

config/promitor/scraper/metrics.yaml

@@ -6,22 +6,13 @@ azureMetadata:
 metricDefaults:
   aggregation:
     interval: 00:05:00
+  labels:
+    geo: china
+    environment: dev
   scraping:
     # Every minute
     schedule: "0 * * ? * *"
 metrics:
-- name: promitor_demo_frontdoor_backend_health
-  description: "Average percentage of memory usage on an Azure App Plan"
-  resourceType: FrontDoor
-  labels:
-    app: promitor
-  azureMetricConfiguration:
-    metricName: BackendHealthPercentage
-    aggregation:
-      type: Average
-  resources:
-  - name: promitor-landscape
-    resourceGroupName: promitor-landscape
 - name: promitor_demo_appplan_percentage_cpu
   description: "Average percentage of memory usage on an Azure App Plan"
   resourceType: AppPlan
@@ -136,6 +127,9 @@ metrics:
 - name: promitor_demo_servicebus_messagecount_discovered
   description: "Average percentage of memory usage on an Azure App Plan"
   resourceType: ServiceBusNamespace
+  labels:
+    geo: europe
+    app: promitor
   azureMetricConfiguration:
     metricName: ActiveMessages
     aggregation:

docs/configuration/v1.x/metrics/index.md

@@ -30,7 +30,6 @@ values are `v1`.
 - `metricDefaults.aggregation.interval` - The default interval which defines over
   what period measurements of a metric should be aggregated.
   a cron that fits your needs.
-- `metricDefaults.labels` - The default lebals that will be applied to all metrics. _(starting as of v1.6)_
 
 ### Metrics
 

docs/configuration/v2.x/metrics/index.md

@@ -30,7 +30,7 @@ values are `v1`. *(Required)*
 - `metricDefaults.aggregation.interval` - The default interval which defines over
   what period measurements of a metric should be aggregated.
   a cron that fits your needs.
-- `metricDefaults.labels` - The default lebals that will be applied to all metrics. _(starting as of v1.6)_
+- `metricDefaults.labels` - The default labels that will be applied to all metrics. _(starting as of v2.3)_
 
 ### Metrics
 

src/Promitor.Core.Scraping/Configuration/Model/MetricDefaults.cs

@@ -1,9 +1,13 @@
-namespace Promitor.Core.Scraping.Configuration.Model
+using System.Collections.Generic;
+
+namespace Promitor.Core.Scraping.Configuration.Model
 {
     public class MetricDefaults
     {
         public Aggregation Aggregation { get; set; } = new Aggregation();
 
         public Scraping Scraping { get; set; } = new Scraping();
+
+        public Dictionary<string, string> Labels { get; set; } = new Dictionary<string, string>();
     }
 }

src/Promitor.Core.Scraping/Configuration/Providers/Interfaces/IMetricsDeclarationProvider.cs

@@ -1,4 +1,5 @@
-using Promitor.Core.Scraping.Configuration.Model;
+using System.Collections.Generic;
+using Promitor.Core.Scraping.Configuration.Model;
 using Promitor.Core.Scraping.Configuration.Model.Metrics;
 using Promitor.Core.Scraping.Configuration.Serialization;
 
@@ -25,7 +26,6 @@ public interface IMetricsDeclarationProvider
         ///     configuration elements to metrics where those values aren't specified. <c>false</c> otherwise
         /// </param>
         /// <param name="errorReporter">Used to report errors during the deserialization process.</param>
-        /// <returns></returns>
         MetricDefinition GetMetricDefinition(string metricName, bool applyDefaults = false, IErrorReporter errorReporter = null);
 
         /// <summary>
@@ -37,9 +37,18 @@ public interface IMetricsDeclarationProvider
         ///     configuration elements to metrics where those values aren't specified. <c>false</c> otherwise
         /// </param>
         /// <param name="errorReporter">Used to report errors during the deserialization process.</param>
-        /// <returns></returns>
         PrometheusMetricDefinition GetPrometheusDefinition(string metricName, bool applyDefaults = false, IErrorReporter errorReporter = null);
 
+        /// <summary>
+        ///     Get default metrics that are defined for all metrics
+        /// </summary>
+        /// <param name="applyDefaults">
+        ///     <c>true</c> if the provider should apply default values from top-level
+        ///     configuration elements to metrics where those values aren't specified. <c>false</c> otherwise
+        /// </param>
+        /// <param name="errorReporter">Used to report errors during the deserialization process.</param>
+        Dictionary<string, string> GetDefaultLabels(bool applyDefaults = false, IErrorReporter errorReporter = null);
+
         /// <summary>
         ///     Gets the serialized metrics declaration
         /// </summary>

src/Promitor.Core.Scraping/Configuration/Providers/MetricsDeclarationProvider.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using Microsoft.Extensions.Configuration;
@@ -23,10 +24,16 @@ public MetricsDeclarationProvider(IConfiguration configuration, ConfigurationSer
 
         public virtual PrometheusMetricDefinition GetPrometheusDefinition(string metricName, bool applyDefaults = false, IErrorReporter errorReporter = null)
         {
-            var foundMetric = GetMetricDefinition(metricName,applyDefaults,errorReporter);
+            var foundMetric = GetMetricDefinition(metricName, applyDefaults, errorReporter);
             return foundMetric?.PrometheusMetricDefinition;
         }
 
+        public virtual Dictionary<string, string> GetDefaultLabels(bool applyDefaults = false, IErrorReporter errorReporter = null)
+        {
+            var metricsDeclaration = Get(applyDefaults, errorReporter);
+            return metricsDeclaration.MetricDefaults?.Labels ?? new Dictionary<string, string>();
+        }
+
         public virtual MetricDefinition GetMetricDefinition(string metricName, bool applyDefaults = false, IErrorReporter errorReporter = null)
         {
             var metricsDeclaration = Get(applyDefaults, errorReporter);

src/Promitor.Core.Scraping/Configuration/Serialization/v1/Core/MetricDefaultsDeserializer.cs

@@ -15,6 +15,7 @@ public MetricDefaultsDeserializer(
             Map(defaults => defaults.Scraping)
                 .IsRequired()
                 .MapUsingDeserializer(scrapingDeserializer);
+            Map(defaults => defaults.Labels);
         }
     }
 }

src/Promitor.Core.Scraping/Configuration/Serialization/v1/Model/MetricDefaultsV1.cs

@@ -1,4 +1,6 @@
-namespace Promitor.Core.Scraping.Configuration.Serialization.v1.Model
+using System.Collections.Generic;
+
+namespace Promitor.Core.Scraping.Configuration.Serialization.v1.Model
 {
     /// <summary>
     /// Contains default settings that apply to all metrics.
@@ -14,5 +16,10 @@ public class MetricDefaultsV1
         /// The default scraping settings.
         /// </summary>
         public ScrapingV1 Scraping { get; set; }
+
+        /// <summary>
+        /// The default metric labels.
+        /// </summary>
+        public Dictionary<string, string> Labels { get; set; } = new Dictionary<string, string>();
     }
 }

src/Promitor.Integrations.Sinks.Prometheus/PrometheusScrapingEndpointMetricSink.cs

@@ -49,8 +49,9 @@ public async Task ReportMetricAsync(string metricName, string metricDescription,
             {
                 var metricValue = DetermineMetricMeasurement(measuredMetric);
                 var metricDefinition = _metricsDeclarationProvider.GetPrometheusDefinition(metricName);
-
-                var metricLabels = DetermineLabels(metricDefinition, scrapeResult, measuredMetric);
+                var defaultLabels = _metricsDeclarationProvider.GetDefaultLabels();
+
+                var metricLabels = DetermineLabels(metricDefinition, scrapeResult, measuredMetric, defaultLabels);
 
                 var reportMetricTask = ReportMetricAsync(metricName, metricDescription, metricValue, metricLabels);
                 reportMetricTasks.Add(reportMetricTask);
@@ -85,7 +86,7 @@ private IMetricFamily<IGauge> CreateGauge(string metricName, string metricDescri
             return gauge;
         }
 
-        private Dictionary<string, string> DetermineLabels(PrometheusMetricDefinition metricDefinition, ScrapeResult scrapeResult, MeasuredMetric measuredMetric)
+        private Dictionary<string, string> DetermineLabels(PrometheusMetricDefinition metricDefinition, ScrapeResult scrapeResult, MeasuredMetric measuredMetric, Dictionary<string, string> defaultLabels)
         {
             var labels = new Dictionary<string, string>(scrapeResult.Labels.Select(label => new KeyValuePair<string, string>(label.Key.SanitizeForPrometheusLabelKey(), label.Value)));
 
@@ -109,8 +110,17 @@ private Dictionary<string, string> DetermineLabels(PrometheusMetricDefinition me
                 }
             }
 
+            foreach (var defaultLabel in defaultLabels)
+            {
+                var defaultLabelKey = defaultLabel.Key.SanitizeForPrometheusLabelKey();
+                if (labels.ContainsKey(defaultLabelKey) == false)
+                {
+                    labels.Add(defaultLabelKey, defaultLabel.Value);
+                }
+            }
+
             // Transform labels, if need be
-            if(_prometheusConfiguration.CurrentValue.Labels!=null)
+            if(_prometheusConfiguration.CurrentValue.Labels != null)
             {
                 labels = LabelTransformer.TransformLabels(_prometheusConfiguration.CurrentValue.Labels.Transformation,labels);
             }

src/Promitor.Tests.Unit/Metrics/Sinks/PrometheusScrapingEndpointMetricSinkTests.cs

@@ -12,6 +12,7 @@
 using Prometheus.Client;
 using Promitor.Core.Metrics;
 using Promitor.Core.Scraping.Configuration.Serialization.v1.Mapping;
+using Promitor.Core.Scraping.Configuration.Serialization.v1.Model;
 using Promitor.Integrations.Sinks.Prometheus;
 using Promitor.Integrations.Sinks.Prometheus.Configuration;
 using Promitor.Tests.Unit.Builders.Metrics.v1;
@@ -86,7 +87,11 @@ public async Task ReportMetricAsync_GetsValidInputWithMetricValue_SuccessfullyWr
             var metricDescription = BogusGenerator.Lorem.Sentence();
             var metricValue = BogusGenerator.Random.Double();
             var scrapeResult = ScrapeResultGenerator.Generate(metricValue);
-            var metricsDeclarationProvider = CreateMetricsDeclarationProvider(metricName);
+            var defaultLabels = new Dictionary<string, string>
+            {
+                {"app", "promitor"}
+            };
+            var metricsDeclarationProvider = CreateMetricsDeclarationProvider(metricName, defaultLabels: defaultLabels);
             var prometheusConfiguration = CreatePrometheusConfiguration();
             var mocks = CreatePrometheusMetricFactoryMock();
             var metricSink = new PrometheusScrapingEndpointMetricSink(mocks.Factory.Object, metricsDeclarationProvider, prometheusConfiguration, NullLogger<PrometheusScrapingEndpointMetricSink>.Instance);
@@ -95,8 +100,8 @@ public async Task ReportMetricAsync_GetsValidInputWithMetricValue_SuccessfullyWr
             await metricSink.ReportMetricAsync(metricName, metricDescription, scrapeResult);
 
             // Assert
-            mocks.Factory.Verify(mock => mock.CreateGauge(metricName, metricDescription, It.IsAny<bool>(), It.Is<string[]>(specified => EnsureAllArrayEntriesAreSpecified(specified, scrapeResult.Labels.Keys.ToArray()))), Times.Once());
-            mocks.MetricFamily.Verify(mock => mock.WithLabels(It.Is<string[]>(specified => EnsureAllArrayEntriesAreSpecified(specified, scrapeResult.Labels.Values.ToArray()))), Times.Once());
+            mocks.Factory.Verify(mock => mock.CreateGauge(metricName, metricDescription, It.IsAny<bool>(), It.Is<string[]>(specified => EnsureAllArrayEntriesAreSpecified(specified, scrapeResult.Labels.Keys.ToArray(), defaultLabels.Keys.ToArray()))), Times.Once());
+            mocks.MetricFamily.Verify(mock => mock.WithLabels(It.Is<string[]>(specified => EnsureAllArrayEntriesAreSpecified(specified, scrapeResult.Labels.Values.ToArray(), defaultLabels.Values.ToArray()))), Times.Once());
             mocks.Gauge.Verify(mock => mock.Set(metricValue), Times.Once());
         }
 
@@ -334,17 +339,33 @@ public async Task ReportMetricAsync_GetsValidInputWithMetricLabelWithSameKeyAsSc
             mocks.Gauge.Verify(mock => mock.Set(metricValue), Times.Once());
         }
 
-        private bool EnsureAllArrayEntriesAreSpecified(string[] specified, string[] expected)
+        private bool EnsureAllArrayEntriesAreSpecified(string[] specified, string[] expectedMetricLabels)
         {
-            if (specified.Length < expected.Length)
+            if (specified.Length < expectedMetricLabels.Length)
             {
                 return false;
             }
 
-            var outcome = Array.Exists(expected, entry=>specified.Contains(entry.ToLower()));
+            var outcome = Array.Exists(expectedMetricLabels, entry => specified.Contains(entry.ToLower()));
             return outcome;
         }
 
+        private bool EnsureAllArrayEntriesAreSpecified(string[] specified, string[] expectedMetricLabels, string[] expectedDefaultLabels)
+        {
+            var expectedTotalLabelCount = expectedDefaultLabels.Length + expectedMetricLabels.Length;
+            if (specified.Length != expectedTotalLabelCount)
+            {
+                return false;
+            }
+
+            var isSuccessful = Array.Exists(expectedMetricLabels, entry => specified.Contains(entry.ToLower()));
+            if(isSuccessful)
+            {
+                isSuccessful = Array.Exists(expectedMetricLabels, entry => specified.Contains(entry.ToLower()));
+            }
+            return isSuccessful;
+        }
+
         private IOptionsMonitor<PrometheusScrapingEndpointSinkConfiguration> CreatePrometheusConfiguration(bool enableMetricsTimestamps = true, double? metricUnavailableValue = -1)
         {
             var prometheusScrapingEndpointSinkConfiguration = new PrometheusScrapingEndpointSinkConfiguration
@@ -357,13 +378,24 @@ private IOptionsMonitor<PrometheusScrapingEndpointSinkConfiguration> CreateProme
             return new OptionsMonitorStub<PrometheusScrapingEndpointSinkConfiguration>(prometheusScrapingEndpointSinkConfiguration);
         }
 
-        private MetricsDeclarationProviderStub CreateMetricsDeclarationProvider(string metricName,  Dictionary<string, string> labels = null)
+        private MetricsDeclarationProviderStub CreateMetricsDeclarationProvider(string metricName,  Dictionary<string, string> labels = null, Dictionary<string, string> defaultLabels = null)
         {
             var mapperConfiguration = new MapperConfiguration(c => c.AddProfile<V1MappingProfile>());
             var mapper = mapperConfiguration.CreateMapper();
-            var rawDeclaration = MetricsDeclarationBuilder.WithMetadata()
-                .WithServiceBusMetric(metricName, labels: labels)
-                .Build(mapper);
+            var metricBuilder = MetricsDeclarationBuilder.WithMetadata();
+
+            if (defaultLabels != null)
+            {
+                var defaults = new MetricDefaultsV1
+                {
+                    Labels = defaultLabels
+                };
+
+                metricBuilder.WithDefaults(defaults);
+            }
+
+            var rawDeclaration = metricBuilder.WithServiceBusMetric(metricName, labels: labels)
+                                                    .Build(mapper);
 
             var metricsDeclarationProvider = new MetricsDeclarationProviderStub(rawDeclaration, mapper);
             return metricsDeclarationProvider;

src/Promitor.Tests.Unit/Serialization/v1/V1SerializationTests.cs

@@ -54,6 +54,10 @@ public V1SerializationTests()
                     Scraping = new ScrapingV1
                     {
                         Schedule = "1 2 3 4 5"
+                    },
+                    Labels = new Dictionary<string, string>
+                    {
+                        {"geo", "china"}
                     }
                 },
                 Metrics = new List<MetricDefinitionV1>
@@ -147,6 +151,7 @@ public void Deserialize_SerializedModel_CanDeserialize()
             Assert.Equal("promitor-group", deserializedModel.AzureMetadata.ResourceGroupName);
             Assert.Equal(TimeSpan.FromMinutes(7), deserializedModel.MetricDefaults.Aggregation.Interval);
             Assert.Equal("1 2 3 4 5", deserializedModel.MetricDefaults.Scraping.Schedule);
+            Assert.Equal("china", deserializedModel.MetricDefaults.Labels["geo"]);
 
             // Check first metric
             Assert.Equal("promitor_demo_generic_queue_size", deserializedModel.Metrics.ElementAt(0).Name);