feat: public preview support for microsoft.azuremonitor.containers.metrics in ARC clusters (managed prometheus)

.gitignore

@@ -88,6 +88,7 @@ venv/
 ENV/
 env/
 env27/
+azenv/
 
 # Spyder project settings
 .spyderproject

src/k8s-extension/HISTORY.rst

@@ -6,6 +6,7 @@ Release History
 1.4.1
 ++++++++++++++++++
 * microsoft.azureml.kubernetes: Fix sslSecret parameter in update operation
+* microsoft.azuremonitor.containers.metrics : public preview support for managed prometheus in ARC clusters
 
 1.4.0
 ++++++++++++++++++
@@ -43,7 +44,7 @@ Release History
 
 1.3.4
 ++++++++++++++++++
-* Fix to address the error TypeError: cf_k8s_extension() takes 1 positional argument but 2 were given while running command az k8s-extension extension-types list 
+* Fix to address the error TypeError: cf_k8s_extension() takes 1 positional argument but 2 were given while running command az k8s-extension extension-types list
 
 1.3.3
 ++++++++++++++++++
@@ -97,9 +98,9 @@ Release History
 1.2.0
 ++++++++++++++++++
 * microsoft.azureml.kubernetes: Update AzureMLKubernetes install parameters on inferenceRouterServiceType and internalLoadBalancerProvider
-* microsoft.openservicemesh: Change extension validation logic osm-arc 
-* microsoft.azuremonitor.containers: Add Managed Identity Auth support for ContainerInsights Extension 
-* microsoft.azuremonitor.containers: Bring back containerInsights solution addition in MSI mode 
+* microsoft.openservicemesh: Change extension validation logic osm-arc
+* microsoft.azuremonitor.containers: Add Managed Identity Auth support for ContainerInsights Extension
+* microsoft.azuremonitor.containers: Bring back containerInsights solution addition in MSI mode
 
 1.1.0
 ++++++++++++++++++
@@ -162,7 +163,7 @@ Release History
 0.5.0
 ++++++++++++++++++
 * Add microsoft.openservicemesh customization to check distros
-* Delete customization for partners 
+* Delete customization for partners
 
 0.4.3
 ++++++++++++++++++
@@ -199,7 +200,7 @@ Release History
 ++++++++++++++++++
 
 * Remove `k8s-extension update` until PATCH is supported
-* Improved logging for overwriting extension name with default 
+* Improved logging for overwriting extension name with default
 
 0.2.0
 ++++++++++++++++++

src/k8s-extension/azext_k8s_extension/custom.py

@@ -24,6 +24,7 @@
 from ._validators import validate_cc_registration
 
 from .partner_extensions.ContainerInsights import ContainerInsights
+from .partner_extensions.AzureMonitorMetrics import AzureMonitorMetrics
 from .partner_extensions.AzureDefender import AzureDefender
 from .partner_extensions.OpenServiceMesh import OpenServiceMesh
 from .partner_extensions.AzureMLKubernetes import AzureMLKubernetes
@@ -44,6 +45,7 @@
 def ExtensionFactory(extension_name):
     extension_map = {
         "microsoft.azuremonitor.containers": ContainerInsights,
+        "microsoft.azuremonitor.containers.metrics": AzureMonitorMetrics,
         "microsoft.azuredefender.kubernetes": AzureDefender,
         "microsoft.openservicemesh": OpenServiceMesh,
         "microsoft.azureml.kubernetes": AzureMLKubernetes,

src/k8s-extension/azext_k8s_extension/partner_extensions/AzureMonitorMetrics.py

@@ -0,0 +1,77 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+# pylint: disable=unused-argument
+
+import datetime
+import json
+import re
+
+from ..utils import get_cluster_rp_api_version
+
+from knack.log import get_logger
+
+from azure.cli.core.commands.client_factory import get_subscription_id
+
+from ..vendored_sdks.models import Extension
+from ..vendored_sdks.models import ScopeCluster
+from ..vendored_sdks.models import Scope
+
+from .DefaultExtension import DefaultExtension
+from .azuremonitormetrics.azuremonitorprofile import ensure_azure_monitor_profile_prerequisites, unlink_azure_monitor_profile_artifacts
+
+from .._client_factory import (
+    cf_resources, cf_resource_groups, cf_log_analytics)
+
+logger = get_logger(__name__)
+
+
+class AzureMonitorMetrics(DefaultExtension):
+    def Create(self, cmd, client, resource_group_name, cluster_name, name, cluster_type, cluster_rp,
+               extension_type, scope, auto_upgrade_minor_version, release_train, version, target_namespace,
+               release_namespace, configuration_settings, configuration_protected_settings,
+               configuration_settings_file, configuration_protected_settings_file,
+               plan_name, plan_publisher, plan_product):
+        """ExtensionType 'microsoft.azuremonitor.containers.metrics' specific validations & defaults for Create
+           Must create and return a valid 'Extension' object.
+
+        """
+        name = 'azuremonitor-metrics'
+        release_namespace = 'kube-system'
+        # Scope is always cluster
+        scope_cluster = ScopeCluster(release_namespace=release_namespace)
+        ext_scope = Scope(cluster=scope_cluster, namespace=None)
+
+        # If release-train is not input, set it to 'stable'
+        if release_train is None:
+            release_train = 'stable'
+
+        cluster_subscription = get_subscription_id(cmd.cli_ctx)
+        ensure_azure_monitor_profile_prerequisites(
+            cmd,
+            cluster_rp,
+            cluster_subscription,
+            resource_group_name,
+            cluster_name,
+            configuration_settings,
+            cluster_type
+        )
+
+        create_identity = True
+        extension = Extension(
+            extension_type=extension_type,
+            auto_upgrade_minor_version=auto_upgrade_minor_version,
+            release_train=release_train,
+            version=version,
+            scope=ext_scope,
+            configuration_settings=configuration_settings,
+            configuration_protected_settings=configuration_protected_settings
+        )
+        return extension, name, create_identity
+
+    def Delete(self, cmd, client, resource_group_name, cluster_name, name, cluster_type, cluster_rp, yes):
+        # cluster_rp, _ = get_cluster_rp_api_version(cluster_type=cluster_type, cluster_rp=cluster_rp)
+        cluster_subscription = get_subscription_id(cmd.cli_ctx)
+        unlink_azure_monitor_profile_artifacts(cmd, cluster_subscription, resource_group_name, cluster_name)

src/k8s-extension/azext_k8s_extension/partner_extensions/azuremonitormetrics/amg/link.py

@@ -0,0 +1,91 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+import json
+import uuid
+from knack.util import CLIError
+from ..constants import (
+    GRAFANA_API,
+    GRAFANA_ROLE_ASSIGNMENT_API,
+    GrafanaLink
+)
+from ..helper import sanitize_resource_id
+
+
+def link_grafana_instance(cmd, azure_monitor_workspace_resource_id, configuration_settings):
+    from azure.cli.core.util import send_raw_request
+    # GET grafana principal ID
+    try:
+        grafana_resource_id = ""
+        if configuration_settings is not None and 'grafana-resource-id' in configuration_settings:
+            grafana_resource_id = configuration_settings['grafana-resource-id']
+        if grafana_resource_id is None or grafana_resource_id == "":
+            return GrafanaLink.NOPARAMPROVIDED
+        grafana_resource_id = sanitize_resource_id(grafana_resource_id)
+        grafanaURI = "{0}{1}?api-version={2}".format(
+            cmd.cli_ctx.cloud.endpoints.resource_manager,
+            grafana_resource_id,
+            GRAFANA_API
+        )
+        headers = ['User-Agent=arc-azuremonitormetrics.link_grafana_instance']
+        grafanaArmResponse = send_raw_request(cmd.cli_ctx, "GET", grafanaURI, body={}, headers=headers)
+        servicePrincipalId = grafanaArmResponse.json()["identity"]["principalId"]
+    except CLIError as e:
+        raise CLIError(e)
+    # Add Role Assignment
+    try:
+        MonitoringDataReader = "b0d8363b-8ddd-447d-831f-62ca05bff136"
+        roleDefinitionURI = "{0}{1}/providers/Microsoft.Authorization/roleAssignments/{2}?api-version={3}".format(
+            cmd.cli_ctx.cloud.endpoints.resource_manager,
+            azure_monitor_workspace_resource_id,
+            uuid.uuid4(),
+            GRAFANA_ROLE_ASSIGNMENT_API
+        )
+        roleDefinitionId = "{0}/providers/Microsoft.Authorization/roleDefinitions/{1}".format(
+            azure_monitor_workspace_resource_id,
+            MonitoringDataReader
+        )
+        association_body = json.dumps({
+            "properties": {
+                "roleDefinitionId": roleDefinitionId,
+                "principalId": servicePrincipalId
+            }
+        })
+        headers = ['User-Agent=arc-azuremonitormetrics.add_role_assignment']
+        send_raw_request(cmd.cli_ctx, "PUT", roleDefinitionURI, body=association_body, headers=headers)
+    except CLIError as e:
+        if e.response.status_code != 409:
+            erroString = "Role Assingment failed. Please manually assign the `Monitoring Data Reader` role\
+                to the Azure Monitor Workspace ({0}) for the Azure Managed Grafana\
+                System Assigned Managed Identity ({1})".format(
+                azure_monitor_workspace_resource_id,
+                servicePrincipalId
+            )
+            print(erroString)
+    # Setting up AMW Integration
+    targetGrafanaArmPayload = grafanaArmResponse.json()
+    if targetGrafanaArmPayload["properties"] is None:
+        raise CLIError("Invalid grafana payload to add AMW integration")
+    if "grafanaIntegrations" not in json.dumps(targetGrafanaArmPayload):
+        targetGrafanaArmPayload["properties"]["grafanaIntegrations"] = {}
+    if "azureMonitorWorkspaceIntegrations" not in json.dumps(targetGrafanaArmPayload):
+        targetGrafanaArmPayload["properties"]["grafanaIntegrations"]["azureMonitorWorkspaceIntegrations"] = []
+    amwIntegrations = targetGrafanaArmPayload["properties"]["grafanaIntegrations"]["azureMonitorWorkspaceIntegrations"]
+    if amwIntegrations != [] and azure_monitor_workspace_resource_id in json.dumps(amwIntegrations).lower():
+        return GrafanaLink.ALREADYPRESENT
+    try:
+        grafanaURI = "{0}{1}?api-version={2}".format(
+            cmd.cli_ctx.cloud.endpoints.resource_manager,
+            grafana_resource_id,
+            GRAFANA_API
+        )
+        targetGrafanaArmPayload["properties"]["grafanaIntegrations"]["azureMonitorWorkspaceIntegrations"].append({
+            "azureMonitorWorkspaceResourceId": azure_monitor_workspace_resource_id
+        })
+        targetGrafanaArmPayload = json.dumps(targetGrafanaArmPayload)
+        headers = ['User-Agent=arc-azuremonitormetrics.setup_amw_grafana_integration', 'Content-Type=application/json']
+        send_raw_request(cmd.cli_ctx, "PUT", grafanaURI, body=targetGrafanaArmPayload, headers=headers)
+    except CLIError as e:
+        raise CLIError(e)
+    return GrafanaLink.SUCCESS

src/k8s-extension/azext_k8s_extension/partner_extensions/azuremonitormetrics/amw/create.py

@@ -0,0 +1,45 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+import json
+from azure.core.exceptions import HttpResponseError
+from knack.util import CLIError
+
+from ..constants import MAC_API
+from .defaults import get_default_mac_name_and_region
+from ...._client_factory import cf_resources, cf_resource_groups
+
+
+def create_default_mac(cmd, cluster_subscription, cluster_region):
+    from azure.cli.core.util import send_raw_request
+    default_mac_name, default_mac_region = get_default_mac_name_and_region(cmd, cluster_region)
+    default_resource_group_name = f"DefaultResourceGroup-{default_mac_region}"
+    azure_monitor_workspace_resource_id = f"/subscriptions/{cluster_subscription}/resourceGroups/{default_resource_group_name}/providers/microsoft.monitor/accounts/{default_mac_name}"
+    # Check if default resource group exists or not, if it does not then create it
+    resource_groups = cf_resource_groups(cmd.cli_ctx, cluster_subscription)
+    resources = cf_resources(cmd.cli_ctx, cluster_subscription)
+
+    if resource_groups.check_existence(default_resource_group_name):
+        try:
+            resource = resources.get_by_id(azure_monitor_workspace_resource_id, MAC_API)
+            # If MAC already exists then return from here
+            # location can have spaces for example 'East US'
+            # and some workspaces it will be "eastus" hence remove the spaces and converting lowercase
+            amw_location = resource.location.replace(" ", "").lower()
+            return azure_monitor_workspace_resource_id, amw_location
+        except HttpResponseError as ex:
+            if ex.status_code != 404:
+                raise ex
+    else:
+        resource_groups.create_or_update(default_resource_group_name, {"location": default_mac_region})
+    association_body = json.dumps({"location": default_mac_region, "properties": {}})
+    armendpoint = cmd.cli_ctx.cloud.endpoints.resource_manager
+    association_url = f"{armendpoint}{azure_monitor_workspace_resource_id}?api-version={MAC_API}"
+    try:
+        headers = ['User-Agent=arc-azuremonitormetrics.create_default_mac']
+        send_raw_request(cmd.cli_ctx, "PUT", association_url,
+                         body=association_body, headers=headers)
+        return azure_monitor_workspace_resource_id, default_mac_region.lower()
+    except CLIError as e:
+        raise e

src/k8s-extension/azext_k8s_extension/partner_extensions/azuremonitormetrics/amw/defaults.py

@@ -0,0 +1,40 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+import json
+from ..deaults import get_default_region
+from ..responseparsers.amwlocationresponseparser import (
+    parseResourceProviderResponseForLocations
+)
+from ..constants import RP_LOCATION_API
+from knack.util import CLIError
+
+
+def get_supported_rp_locations(cmd, rp_name):
+    from azure.cli.core.util import send_raw_request
+    supported_locations = []
+    headers = ['User-Agent=arc-azuremonitormetrics.get_supported_rp_locations']
+    armendpoint = cmd.cli_ctx.cloud.endpoints.resource_manager
+    association_url = f"{armendpoint}/providers/{rp_name}?api-version={RP_LOCATION_API}"
+    r = send_raw_request(cmd.cli_ctx, "GET", association_url, headers=headers)
+    data = json.loads(r.text)
+    supported_locations = parseResourceProviderResponseForLocations(data)
+    return supported_locations
+
+
+def get_default_mac_region(cmd, cluster_region):
+    supported_locations = get_supported_rp_locations(cmd, 'Microsoft.Monitor')
+    if cluster_region in supported_locations:
+        return cluster_region
+    if len(supported_locations) > 0:
+        return supported_locations[0]
+    # default to eastus/usgovvirginia based on cloud (mooncake not supported yet)
+    return get_default_region(cmd)
+
+
+def get_default_mac_name_and_region(cmd, cluster_region):
+    default_mac_region = get_default_mac_region(cmd, cluster_region)
+    default_mac_name = "DefaultAzureMonitorWorkspace-" + default_mac_region
+    default_mac_name = default_mac_name[0:43]
+    return default_mac_name, default_mac_region

src/k8s-extension/azext_k8s_extension/partner_extensions/azuremonitormetrics/amw/helper.py

@@ -0,0 +1,39 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+from azure.core.exceptions import HttpResponseError
+from .create import create_default_mac
+from ..helper import sanitize_resource_id
+from ..constants import MAC_API
+from ...._client_factory import cf_resources
+
+
+def get_amw_region(cmd, azure_monitor_workspace_resource_id):
+    # Region of MAC can be different from region of RG so find the location of the azure_monitor_workspace_resource_id
+    amw_subscription_id = azure_monitor_workspace_resource_id.split("/")[2]
+    resources = cf_resources(cmd.cli_ctx, amw_subscription_id)
+    try:
+        resource = resources.get_by_id(
+            azure_monitor_workspace_resource_id, MAC_API)
+        amw_location = resource.location.replace(" ", "").lower()
+        return amw_location
+    except HttpResponseError as ex:
+        raise ex
+
+
+def get_azure_monitor_workspace_resource(cmd, cluster_subscription, cluster_region, configuration_settings):
+    azure_monitor_workspace_resource_id = ""
+    if 'azure-monitor-workspace-resource-id' in configuration_settings:
+        azure_monitor_workspace_resource_id = configuration_settings['azure-monitor-workspace-resource-id']
+    if azure_monitor_workspace_resource_id is None or azure_monitor_workspace_resource_id == "":
+        azure_monitor_workspace_resource_id, azure_monitor_workspace_location = create_default_mac(
+            cmd,
+            cluster_subscription,
+            cluster_region
+        )
+    else:
+        azure_monitor_workspace_resource_id = sanitize_resource_id(azure_monitor_workspace_resource_id)
+        azure_monitor_workspace_location = get_amw_region(cmd, azure_monitor_workspace_resource_id)
+    print(f"Using Azure Monitor Workspace (stores prometheus metrics) : {azure_monitor_workspace_resource_id}")
+    return azure_monitor_workspace_resource_id, azure_monitor_workspace_location