From 1b07284873b3f784b66bb082545eb8e38afa8f12 Mon Sep 17 00:00:00 2001 From: Tom Kerkhove Date: Mon, 6 Jan 2020 17:54:11 +0100 Subject: [PATCH 1/7] Provide support for Azure cloud other than Global Signed-off-by: Tom Kerkhove --- .../Configuration/Model/AzureMetadata.cs | 5 +- .../v1/Core/AzureMetadataDeserializer.cs | 35 +++++++++++- .../Serialization/v1/Model/AzureCloudsV1.cs | 10 ++++ .../Serialization/v1/Model/AzureMetadataV1.cs | 5 +- .../Factories/MetricScraperFactory.cs | 2 +- .../AzureMonitorClient.cs | 5 +- .../v1/Core/AzureMetadataDeserializerTests.cs | 57 ++++++++++++++++++- 7 files changed, 110 insertions(+), 9 deletions(-) create mode 100644 src/Promitor.Core.Scraping/Configuration/Serialization/v1/Model/AzureCloudsV1.cs diff --git a/src/Promitor.Core.Scraping/Configuration/Model/AzureMetadata.cs b/src/Promitor.Core.Scraping/Configuration/Model/AzureMetadata.cs index 569e6c4a6..3473f28f8 100644 --- a/src/Promitor.Core.Scraping/Configuration/Model/AzureMetadata.cs +++ b/src/Promitor.Core.Scraping/Configuration/Model/AzureMetadata.cs @@ -1,9 +1,12 @@ -namespace Promitor.Core.Scraping.Configuration.Model +using Microsoft.Azure.Management.ResourceManager.Fluent; + +namespace Promitor.Core.Scraping.Configuration.Model { public class AzureMetadata { public string ResourceGroupName { get; set; } public string SubscriptionId { get; set; } public string TenantId { get; set; } + public AzureEnvironment Cloud { get; set; } } } \ No newline at end of file diff --git a/src/Promitor.Core.Scraping/Configuration/Serialization/v1/Core/AzureMetadataDeserializer.cs b/src/Promitor.Core.Scraping/Configuration/Serialization/v1/Core/AzureMetadataDeserializer.cs index 3a7993d66..c4d3927f3 100644 --- a/src/Promitor.Core.Scraping/Configuration/Serialization/v1/Core/AzureMetadataDeserializer.cs +++ b/src/Promitor.Core.Scraping/Configuration/Serialization/v1/Core/AzureMetadataDeserializer.cs @@ -1,4 +1,8 @@ -using Microsoft.Extensions.Logging; +using System; +using System.Linq; +using System.Runtime.Serialization; +using Microsoft.Azure.Management.ResourceManager.Fluent; +using Microsoft.Extensions.Logging; using Promitor.Core.Scraping.Configuration.Serialization.v1.Model; using YamlDotNet.RepresentationModel; @@ -6,6 +10,7 @@ namespace Promitor.Core.Scraping.Configuration.Serialization.v1.Core { public class AzureMetadataDeserializer : Deserializer { + private const string CloudTag = "cloud"; private const string TenantIdTag = "tenantId"; private const string SubscriptionIdTag = "subscriptionId"; private const string ResourceGroupNameTag = "resourceGroupName"; @@ -18,11 +23,37 @@ public override AzureMetadataV1 Deserialize(YamlMappingNode node) { var metadata = new AzureMetadataV1(); + var azureCloud = node.GetEnum(CloudTag); ; + var cloud = DetermineAzureCloud(azureCloud); + metadata.TenantId = node.GetString(TenantIdTag); metadata.SubscriptionId = node.GetString(SubscriptionIdTag); metadata.ResourceGroupName = node.GetString(ResourceGroupNameTag); - + metadata.Cloud = cloud; + return metadata; } + + private AzureEnvironment DetermineAzureCloud(AzureCloudsV1? azureCloud) + { + if (azureCloud == null) + { + return AzureEnvironment.AzureGlobalCloud; + } + + switch (azureCloud) + { + case AzureCloudsV1.Global: + return AzureEnvironment.AzureGlobalCloud; + case AzureCloudsV1.China: + return AzureEnvironment.AzureChinaCloud; + case AzureCloudsV1.Germany: + return AzureEnvironment.AzureGermanCloud; + case AzureCloudsV1.USGov: + return AzureEnvironment.AzureUSGovernment; + default: + throw new ArgumentOutOfRangeException(nameof(azureCloud), azureCloud, $"{azureCloud} is not supported yet"); + } + } } } diff --git a/src/Promitor.Core.Scraping/Configuration/Serialization/v1/Model/AzureCloudsV1.cs b/src/Promitor.Core.Scraping/Configuration/Serialization/v1/Model/AzureCloudsV1.cs new file mode 100644 index 000000000..f12fa4756 --- /dev/null +++ b/src/Promitor.Core.Scraping/Configuration/Serialization/v1/Model/AzureCloudsV1.cs @@ -0,0 +1,10 @@ +namespace Promitor.Core.Scraping.Configuration.Serialization.v1.Model +{ + public enum AzureCloudsV1 + { + Global, + China, + USGov, + Germany + } +} \ No newline at end of file diff --git a/src/Promitor.Core.Scraping/Configuration/Serialization/v1/Model/AzureMetadataV1.cs b/src/Promitor.Core.Scraping/Configuration/Serialization/v1/Model/AzureMetadataV1.cs index cdaf01c9f..b65104cb1 100644 --- a/src/Promitor.Core.Scraping/Configuration/Serialization/v1/Model/AzureMetadataV1.cs +++ b/src/Promitor.Core.Scraping/Configuration/Serialization/v1/Model/AzureMetadataV1.cs @@ -1,9 +1,12 @@ -namespace Promitor.Core.Scraping.Configuration.Serialization.v1.Model +using Microsoft.Azure.Management.ResourceManager.Fluent; + +namespace Promitor.Core.Scraping.Configuration.Serialization.v1.Model { public class AzureMetadataV1 { public string TenantId { get; set; } public string SubscriptionId { get; set; } public string ResourceGroupName { get; set; } + public AzureEnvironment Cloud { get; set; } } } diff --git a/src/Promitor.Core.Scraping/Factories/MetricScraperFactory.cs b/src/Promitor.Core.Scraping/Factories/MetricScraperFactory.cs index 17069e3a6..6a749693e 100644 --- a/src/Promitor.Core.Scraping/Factories/MetricScraperFactory.cs +++ b/src/Promitor.Core.Scraping/Factories/MetricScraperFactory.cs @@ -73,7 +73,7 @@ public IScraper CreateScraper(ResourceType metricDefini private AzureMonitorClient CreateAzureMonitorClient(AzureMetadata azureMetadata, IRuntimeMetricsCollector runtimeMetricsCollector) { var azureCredentials = DetermineAzureCredentials(); - var azureMonitorClient = new AzureMonitorClient(azureMetadata.TenantId, azureMetadata.SubscriptionId, azureCredentials.ApplicationId, azureCredentials.Secret, runtimeMetricsCollector, _logger); + var azureMonitorClient = new AzureMonitorClient(azureMetadata.Cloud,azureMetadata.TenantId, azureMetadata.SubscriptionId, azureCredentials.ApplicationId, azureCredentials.Secret, runtimeMetricsCollector, _logger); return azureMonitorClient; } diff --git a/src/Promitor.Integrations.AzureMonitor/AzureMonitorClient.cs b/src/Promitor.Integrations.AzureMonitor/AzureMonitorClient.cs index 6e0972e41..30bba7cb9 100644 --- a/src/Promitor.Integrations.AzureMonitor/AzureMonitorClient.cs +++ b/src/Promitor.Integrations.AzureMonitor/AzureMonitorClient.cs @@ -24,20 +24,21 @@ public class AzureMonitorClient /// /// Constructor /// + /// Name of the Azure cloud to interact with /// Id of the tenant that owns the Azure subscription /// Id of the Azure subscription /// Id of the Azure AD application used to authenticate with Azure Monitor /// Secret to authenticate with Azure Monitor for the specified Azure AD application /// Metrics collector for our runtime /// Logger to use during interaction with Azure Monitor - public AzureMonitorClient(string tenantId, string subscriptionId, string applicationId, string applicationSecret, IRuntimeMetricsCollector runtimeMetricsCollector, ILogger logger) + public AzureMonitorClient(AzureEnvironment azureCloud, string tenantId, string subscriptionId, string applicationId, string applicationSecret, IRuntimeMetricsCollector runtimeMetricsCollector, ILogger logger) { Guard.NotNullOrWhitespace(tenantId, nameof(tenantId)); Guard.NotNullOrWhitespace(subscriptionId, nameof(subscriptionId)); Guard.NotNullOrWhitespace(applicationId, nameof(applicationId)); Guard.NotNullOrWhitespace(applicationSecret, nameof(applicationSecret)); - var credentials = _azureCredentialsFactory.FromServicePrincipal(applicationId, applicationSecret, tenantId, AzureEnvironment.AzureGlobalCloud); + var credentials = _azureCredentialsFactory.FromServicePrincipal(applicationId, applicationSecret, tenantId, azureCloud); var monitorHandler = new AzureResourceManagerThrottlingRequestHandler(tenantId, subscriptionId, applicationId, runtimeMetricsCollector, logger); _authenticatedAzureSubscription = Azure.Configure().WithDelegatingHandler(monitorHandler).Authenticate(credentials).WithSubscription(subscriptionId); diff --git a/src/Promitor.Scraper.Tests.Unit/Serialization/v1/Core/AzureMetadataDeserializerTests.cs b/src/Promitor.Scraper.Tests.Unit/Serialization/v1/Core/AzureMetadataDeserializerTests.cs index 0bed9775d..a5086cf69 100644 --- a/src/Promitor.Scraper.Tests.Unit/Serialization/v1/Core/AzureMetadataDeserializerTests.cs +++ b/src/Promitor.Scraper.Tests.Unit/Serialization/v1/Core/AzureMetadataDeserializerTests.cs @@ -1,7 +1,12 @@ -using System.ComponentModel; +using System; +using System.ComponentModel; +using System.Runtime.Serialization; +using Microsoft.Azure.Management.ResourceManager.Fluent; using Microsoft.Extensions.Logging.Abstractions; using Promitor.Core.Scraping.Configuration.Serialization.v1.Core; +using Promitor.Core.Scraping.Configuration.Serialization.v1.Model; using Xunit; +using YamlDotNet.RepresentationModel; namespace Promitor.Scraper.Tests.Unit.Serialization.v1.Core { @@ -15,13 +20,61 @@ public AzureMetadataDeserializerTests() _deserializer = new AzureMetadataDeserializer(NullLogger.Instance); } + [Fact] + public void Deserialize_AzureCloudSupplied_SetsAzureCloud() + { + AzureEnvironment azureCloud = AzureEnvironment.AzureChinaCloud; + + var yamlText = + $@"azureMetadata: + cloud: '{AzureCloudsV1.China}'"; + + YamlAssert.PropertySet( + _deserializer, + yamlText, + "azureMetadata", + azureCloud, + a => a.Cloud); + } + + [Fact] + public void Deserialize_AzureCloudNotSupplied_SetsGlobalAzureCloud() + { + AzureEnvironment azureCloud = AzureEnvironment.AzureGlobalCloud; + + var yamlText = + @"azureMetadata: + tenantId: ABC"; + + YamlAssert.PropertySet( + _deserializer, + yamlText, + "azureMetadata", + azureCloud, + a => a.Cloud); + } + + [Fact] + public void Deserialize_InvalidAzureCloudSupplied_ThrowsException() + { + var yamlText = + @"azureMetadata: + cloud: invalid"; + + // Arrange + var node = YamlUtils.CreateYamlNode(yamlText).Children["azureMetadata"]; + + // Act + Assert.Throws(() => _deserializer.Deserialize((YamlMappingNode) node)); + } + [Fact] public void Deserialize_TenantIdSupplied_SetsTenantId() { const string tenantId = "c8819874-9e56-4e3f-b1a8-1c0325138f27"; var yamlText = -$@"azureMetadata: + $@"azureMetadata: tenantId: '{tenantId}'"; YamlAssert.PropertySet( From 739945c66df6aadab49e8d41a396bd07f2c41ad7 Mon Sep 17 00:00:00 2001 From: Tom Kerkhove Date: Tue, 7 Jan 2020 08:13:02 +0100 Subject: [PATCH 2/7] Document support for other Azure clouds Signed-off-by: Tom Kerkhove --- docs/configuration/v1.x/metrics/index.md | 3 +++ docs/faq.md | 6 ++++++ docs/index.md | 3 ++- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/configuration/v1.x/metrics/index.md b/docs/configuration/v1.x/metrics/index.md index 66ab0dd2a..ceeff3fc4 100644 --- a/docs/configuration/v1.x/metrics/index.md +++ b/docs/configuration/v1.x/metrics/index.md @@ -18,6 +18,8 @@ values are `v1`. - `azureMetadata.tenantId` - The id of the Azure tenant that will be queried. - `azureMetadata.subscriptionId` - The id of the default subscription to query. - `azureMetadata.resourceGroupName` - The name of the default resource group to query. +- `azureMetadata.cloud` - The name of the Azure cloud to use. Options are `Global` + (default), `China`, `UsGov` & `Germany`. ### Metric Defaults @@ -70,6 +72,7 @@ azureMetadata: tenantId: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx subscriptionId: yyyyyyyy-yyyy-yyyy-yyyy-yyyyyyyyyyyy resourceGroupName: promitor + cloud: China metricDefaults: aggregation: interval: 00:05:00 diff --git a/docs/faq.md b/docs/faq.md index 5b0148ae3..ec648f1e6 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -3,6 +3,12 @@ layout: default title: Frequently asked questions (FAQs) --- +## What Azure clouds are supported? + +We support `Global` (default), `China`, `UsGov` & `Germany` Azure clouds which can be configured in the metric + configuration under `azureMetadata`. +For more information see our ['Metric Configuration' page](/configuration/v1.x/metrics/#metrics). + ## Are multi-dimensional metrics supported? Yes, every scraper supports scraping multi-dimensional metrics except for diff --git a/docs/index.md b/docs/index.md index d92038c6b..11dcc771b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -37,7 +37,8 @@ Docker image is available on [Docker Hub](https://hub.docker.com/r/tomkerkhove/p - Built-in support for a variety of Azure services ([overview](configuration/v1.x/metrics#supported-azure-services)) - Easy to declare metrics to scrape via YAML & APIs - Easily deployable via Docker & Kubernetes -- Sends telemetry to Azure Application Insights +- Sends telemetry to container logs & Azure Application Insights +- Support for all Azure clouds And there is more on the way - Check our [backlog](https://github.com/tomkerkhove/promitor/issues) and vote for features! From 121bbd0d783eaf309dc5f722a863581e1abf168f Mon Sep 17 00:00:00 2001 From: Tom Kerkhove Date: Tue, 7 Jan 2020 08:17:07 +0100 Subject: [PATCH 3/7] Document new feature in changelog Signed-off-by: Tom Kerkhove --- changelog/content/experimental/unreleased.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/changelog/content/experimental/unreleased.md b/changelog/content/experimental/unreleased.md index dc56eb32f..dff26c235 100644 --- a/changelog/content/experimental/unreleased.md +++ b/changelog/content/experimental/unreleased.md @@ -5,7 +5,8 @@ weight: 1 version: --- -- {{% tag added %}} Multi-dimensional metric support ([docs](https://promitor.io/configuration/v1.x/metrics/#metrics) | [#81](https://github.com/tomkerkhove/promitor/issues/81)) +- {{% tag added %}} Support for all Azure clouds ([docs](https://promitor.io/configuration/v1.x/metrics/#azure) | [FAQ](https://promitor.io/faq) | [#114](https://github.com/tomkerkhove/promitor/issues/114)) +- {{% tag added %}} Multi-dimensional metric support ([docs](https://promitor.io/configuration/v1.x/metrics/#metrics) | [FAQ](https://promitor.io/faq) | [#81](https://github.com/tomkerkhove/promitor/issues/81)) - {{% tag added %}} Azure SQL Database Scraper ([docs](https://promitor.io/configuration/v1.x/metrics/sql-database) | [#317](https://github.com/tomkerkhove/promitor/issues/317)) - {{% tag added %}} Azure SQL Managed Instance Scraper ([docs](https://promitor.io/configuration/v1.x/metrics/sql-managed-instance) | [#381](https://github.com/tomkerkhove/promitor/issues/381)) - {{% tag added %}} OpenAPI v3.0 support (`/api/v1/docs.json` | [docs](ttps://promitor.io/operations/#exploring-our-rest-apis) | [#734](https://github.com/tomkerkhove/promitor/issues/734)) From 38d04d625f13cc342ed03b686d7f743cd9c37140 Mon Sep 17 00:00:00 2001 From: Tom Kerkhove Date: Tue, 7 Jan 2020 08:17:17 +0100 Subject: [PATCH 4/7] Improve FAQ formatting Signed-off-by: Tom Kerkhove --- docs/faq.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/faq.md b/docs/faq.md index ec648f1e6..7658211b1 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -5,8 +5,10 @@ title: Frequently asked questions (FAQs) ## What Azure clouds are supported? -We support `Global` (default), `China`, `UsGov` & `Germany` Azure clouds which can be configured in the metric - configuration under `azureMetadata`. +We support `Global` (default), `China`, `UsGov` & `Germany` Azure clouds. + +This can be configured in the metric configuration under `azureMetadata`. + For more information see our ['Metric Configuration' page](/configuration/v1.x/metrics/#metrics). ## Are multi-dimensional metrics supported? From 8dfaa903a23cf8322ffeb38b044d870c194775c7 Mon Sep 17 00:00:00 2001 From: Tom Kerkhove Date: Tue, 7 Jan 2020 08:17:35 +0100 Subject: [PATCH 5/7] USGov to UsGov Signed-off-by: Tom Kerkhove --- .../Serialization/v1/Core/AzureMetadataDeserializer.cs | 2 +- .../Configuration/Serialization/v1/Model/AzureCloudsV1.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Promitor.Core.Scraping/Configuration/Serialization/v1/Core/AzureMetadataDeserializer.cs b/src/Promitor.Core.Scraping/Configuration/Serialization/v1/Core/AzureMetadataDeserializer.cs index c4d3927f3..3ff05ced0 100644 --- a/src/Promitor.Core.Scraping/Configuration/Serialization/v1/Core/AzureMetadataDeserializer.cs +++ b/src/Promitor.Core.Scraping/Configuration/Serialization/v1/Core/AzureMetadataDeserializer.cs @@ -49,7 +49,7 @@ private AzureEnvironment DetermineAzureCloud(AzureCloudsV1? azureCloud) return AzureEnvironment.AzureChinaCloud; case AzureCloudsV1.Germany: return AzureEnvironment.AzureGermanCloud; - case AzureCloudsV1.USGov: + case AzureCloudsV1.UsGov: return AzureEnvironment.AzureUSGovernment; default: throw new ArgumentOutOfRangeException(nameof(azureCloud), azureCloud, $"{azureCloud} is not supported yet"); diff --git a/src/Promitor.Core.Scraping/Configuration/Serialization/v1/Model/AzureCloudsV1.cs b/src/Promitor.Core.Scraping/Configuration/Serialization/v1/Model/AzureCloudsV1.cs index f12fa4756..7c386249c 100644 --- a/src/Promitor.Core.Scraping/Configuration/Serialization/v1/Model/AzureCloudsV1.cs +++ b/src/Promitor.Core.Scraping/Configuration/Serialization/v1/Model/AzureCloudsV1.cs @@ -4,7 +4,7 @@ public enum AzureCloudsV1 { Global, China, - USGov, + UsGov, Germany } } \ No newline at end of file From 5292ba1b369262e19dc911267fa10187c0c4674a Mon Sep 17 00:00:00 2001 From: Tom Kerkhove Date: Tue, 7 Jan 2020 08:18:52 +0100 Subject: [PATCH 6/7] Fix code quality Signed-off-by: Tom Kerkhove --- .../Serialization/v1/Core/AzureMetadataDeserializer.cs | 4 +--- .../Serialization/v1/Core/AzureMetadataDeserializerTests.cs | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/Promitor.Core.Scraping/Configuration/Serialization/v1/Core/AzureMetadataDeserializer.cs b/src/Promitor.Core.Scraping/Configuration/Serialization/v1/Core/AzureMetadataDeserializer.cs index 3ff05ced0..ccea0a97f 100644 --- a/src/Promitor.Core.Scraping/Configuration/Serialization/v1/Core/AzureMetadataDeserializer.cs +++ b/src/Promitor.Core.Scraping/Configuration/Serialization/v1/Core/AzureMetadataDeserializer.cs @@ -1,6 +1,4 @@ using System; -using System.Linq; -using System.Runtime.Serialization; using Microsoft.Azure.Management.ResourceManager.Fluent; using Microsoft.Extensions.Logging; using Promitor.Core.Scraping.Configuration.Serialization.v1.Model; @@ -23,7 +21,7 @@ public override AzureMetadataV1 Deserialize(YamlMappingNode node) { var metadata = new AzureMetadataV1(); - var azureCloud = node.GetEnum(CloudTag); ; + var azureCloud = node.GetEnum(CloudTag); var cloud = DetermineAzureCloud(azureCloud); metadata.TenantId = node.GetString(TenantIdTag); diff --git a/src/Promitor.Scraper.Tests.Unit/Serialization/v1/Core/AzureMetadataDeserializerTests.cs b/src/Promitor.Scraper.Tests.Unit/Serialization/v1/Core/AzureMetadataDeserializerTests.cs index a5086cf69..3825c5e82 100644 --- a/src/Promitor.Scraper.Tests.Unit/Serialization/v1/Core/AzureMetadataDeserializerTests.cs +++ b/src/Promitor.Scraper.Tests.Unit/Serialization/v1/Core/AzureMetadataDeserializerTests.cs @@ -1,6 +1,5 @@ using System; using System.ComponentModel; -using System.Runtime.Serialization; using Microsoft.Azure.Management.ResourceManager.Fluent; using Microsoft.Extensions.Logging.Abstractions; using Promitor.Core.Scraping.Configuration.Serialization.v1.Core; From b2439b1a199de6c760ae960dd5c9e8836bfc9efb Mon Sep 17 00:00:00 2001 From: Tom Kerkhove Date: Tue, 7 Jan 2020 08:19:05 +0100 Subject: [PATCH 7/7] Increase allowed Markdown line length Signed-off-by: Tom Kerkhove --- .markdownlint.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.markdownlint.json b/.markdownlint.json index 503463328..6798ee46f 100644 --- a/.markdownlint.json +++ b/.markdownlint.json @@ -1,6 +1,6 @@ { "default": true, - "MD013": { "code_blocks": false, "tables": false }, + "MD013": { "line_length": 120, "code_blocks": false, "tables": false }, "MD041": false, "MD026": false, "MD033": false,