[AKS] Support use custom kubelet identity (#3249)

linter_exclusions.yml

@@ -82,6 +82,9 @@ aks create:
     enable_pod_identity_with_kubenet:
       rule_exclusions:
       - option_length_too_long
+    assign_kubelet_identity:
+      rule_exclusions:
+      - option_length_too_long
 aks enable-addons:
   parameters:
     appgw_watch_namespace:

src/aks-preview/azext_aks_preview/_consts.py

@@ -76,3 +76,6 @@
     'gitops': 'gitops',
     'azure-keyvault-secrets-provider': CONST_AZURE_KEYVAULT_SECRETS_PROVIDER_ADDON_NAME
 }
+
+CONST_MANAGED_IDENTITY_OPERATOR_ROLE = 'Managed Identity Operator'
+CONST_MANAGED_IDENTITY_OPERATOR_ROLE_ID = 'f1a07417-d97a-45cb-824c-7a7467783830'

src/aks-preview/azext_aks_preview/_help.py

@@ -265,7 +265,10 @@
           short-summary: Using managed identity to manage cluster resource group. Default value is true, you can explicitly specify "--client-id" and "--secret" to disable managed identity.
         - name: --assign-identity
           type: string
-          short-summary: (PREVIEW) Specify an existing user assigned identity to manage cluster resource group.
+          short-summary: Specify an existing user assigned identity to manage cluster resource group.
+        - name: --assign-kubelet-identity
+          type: string
+          short-summary: Specify an existing user assigned identity for kubelet's usage, which is typically used to pull image from ACR.
         - name: --api-server-authorized-ip-ranges
           type: string
           short-summary: Comma seperated list of authorized apiserver IP ranges. Set to 0.0.0.0/32 to restrict apiserver traffic to node pools.
@@ -359,6 +362,8 @@
           text: az aks create -g MyResourceGroup -n MyManagedCluster --tags "foo=bar" "baz=qux"
         - name: Create a kubernetes cluster with EncryptionAtHost enabled.
           text: az aks create -g MyResourceGroup -n MyManagedCluster --enable-encryption-at-host
+        - name: Create a kubernetes cluster with custom control plane identity and kubelet identity.
+          text: az aks create -g MyResourceGroup -n MyManagedCluster --assign-identity <control-plane-identity-resource-id> --assign-kubelet-identity <kubelet-identity-resource-id>
 
 """.format(sp_cache=AKS_SERVICE_PRINCIPAL_CACHE)
 

src/aks-preview/azext_aks_preview/_params.py

@@ -23,7 +23,7 @@
     validate_taints, validate_priority, validate_eviction_policy, validate_spot_max_price, validate_acr, validate_user,
     validate_load_balancer_outbound_ports, validate_load_balancer_idle_timeout, validate_nodepool_tags,
     validate_nodepool_labels, validate_vnet_subnet_id, validate_pod_subnet_id, validate_max_surge, validate_assign_identity, validate_addons,
-    validate_pod_identity_pod_labels, validate_pod_identity_resource_name, validate_pod_identity_resource_namespace)
+    validate_pod_identity_pod_labels, validate_pod_identity_resource_name, validate_pod_identity_resource_namespace, validate_assign_kubelet_identity)
 from ._consts import CONST_OUTBOUND_TYPE_LOAD_BALANCER, \
     CONST_OUTBOUND_TYPE_USER_DEFINED_ROUTING, CONST_SCALE_SET_PRIORITY_REGULAR, CONST_SCALE_SET_PRIORITY_SPOT, \
     CONST_SPOT_EVICTION_POLICY_DELETE, CONST_SPOT_EVICTION_POLICY_DEALLOCATE, \
@@ -127,6 +127,7 @@ def load_arguments(self, _):
         c.argument('aci_subnet_name', type=str)
         c.argument('enable_encryption_at_host', arg_type=get_three_state_flag(), help='Enable EncryptionAtHost.')
         c.argument('enable_secret_rotation', action='store_true')
+        c.argument('assign_kubelet_identity', type=str, validator=validate_assign_kubelet_identity)
         c.argument('yes', options_list=['--yes', '-y'], help='Do not prompt for confirmation.', action='store_true')
 
     with self.argument_context('aks update') as c:

src/aks-preview/azext_aks_preview/_validators.py

@@ -468,3 +468,12 @@ def validate_pod_identity_resource_namespace(namespace):
     if not namespace_value:
         # namespace cannot be empty
         raise CLIError('--namespace is required')
+
+
+def validate_assign_kubelet_identity(namespace):
+    if namespace.assign_kubelet_identity is not None:
+        if namespace.assign_kubelet_identity == '':
+            return
+        from msrestazure.tools import is_valid_resource_id
+        if not is_valid_resource_id(namespace.assign_kubelet_identity):
+            raise CLIError("--assign-kubelet-identity is not a valid Azure resource ID.")

src/aks-preview/azext_aks_preview/custom.py

@@ -81,7 +81,8 @@
                                                                       ManagedClusterPodIdentity,
                                                                       ManagedClusterPodIdentityException,
                                                                       UserAssignedIdentity,
-                                                                      RunCommandRequest)
+                                                                      RunCommandRequest,
+                                                                      ManagedClusterPropertiesIdentityProfileValue)
 from ._client_factory import cf_resource_groups
 from ._client_factory import get_auth_management_client
 from ._client_factory import get_graph_rbac_management_client
@@ -112,6 +113,7 @@
 from ._consts import CONST_CONFCOM_ADDON_NAME, CONST_ACC_SGX_QUOTE_HELPER_ENABLED
 from ._consts import CONST_OPEN_SERVICE_MESH_ADDON_NAME
 from ._consts import CONST_AZURE_KEYVAULT_SECRETS_PROVIDER_ADDON_NAME, CONST_SECRET_ROTATION_ENABLED
+from ._consts import CONST_MANAGED_IDENTITY_OPERATOR_ROLE, CONST_MANAGED_IDENTITY_OPERATOR_ROLE_ID
 from ._consts import ADDONS
 from .maintenanceconfiguration import aks_maintenanceconfiguration_update_internal
 from ._consts import CONST_PRIVATE_DNS_ZONE_SYSTEM, CONST_PRIVATE_DNS_ZONE_NONE
@@ -642,6 +644,10 @@ def _get_user_assigned_identity_client_id(cli_ctx, resource_id):
     return _get_user_assigned_identity(cli_ctx, resource_id).client_id
 
 
+def _get_user_assigned_identity_object_id(cli_ctx, resource_id):
+    return _get_user_assigned_identity(cli_ctx, resource_id).principal_id
+
+
 def _update_dict(dict1, dict2):
     cp = dict1.copy()
     cp.update(dict2)
@@ -1027,6 +1033,7 @@ def aks_create(cmd,     # pylint: disable=too-many-locals,too-many-statements,to
                enable_encryption_at_host=False,
                enable_secret_rotation=False,
                no_wait=False,
+               assign_kubelet_identity=None,
                yes=False):
     if not no_ssh_key:
         try:
@@ -1327,6 +1334,22 @@ def aks_create(cmd,     # pylint: disable=too-many-locals,too-many-statements,to
             user_assigned_identities=user_assigned_identity
         )
 
+    identity_profile = None
+    if assign_kubelet_identity:
+        if not assign_identity:
+            raise CLIError('--assign-kubelet-identity can only be specified when --assign-identity is specified')
+        kubelet_identity = _get_user_assigned_identity(cmd.cli_ctx, assign_kubelet_identity)
+        identity_profile = {
+            'kubeletidentity': ManagedClusterPropertiesIdentityProfileValue(
+                resource_id=assign_kubelet_identity,
+                client_id=kubelet_identity.client_id,
+                object_id=kubelet_identity.principal_id
+            )
+        }
+        cluster_identity_object_id = _get_user_assigned_identity_object_id(cmd.cli_ctx, assign_identity)
+        # ensure the cluster identity has "Managed Identity Operator" role at the scope of kubelet identity
+        _ensure_cluster_identity_permission_on_kubelet_identity(cmd.cli_ctx, cluster_identity_object_id, assign_kubelet_identity)
+
     pod_identity_profile = None
     if enable_pod_identity:
         if not enable_managed_identity:
@@ -1363,7 +1386,8 @@ def aks_create(cmd,     # pylint: disable=too-many-locals,too-many-statements,to
         disk_encryption_set_id=node_osdisk_diskencryptionset_id,
         api_server_access_profile=api_server_access_profile,
         auto_upgrade_profile=auto_upgrade_profile,
-        pod_identity_profile=pod_identity_profile)
+        pod_identity_profile=pod_identity_profile,
+        identity_profile=identity_profile)
 
     if node_resource_group:
         mc.node_resource_group = node_resource_group
@@ -3893,9 +3917,6 @@ def _update_addon_pod_identity(instance, enable, pod_identities=None, pod_identi
 
 
 def _ensure_managed_identity_operator_permission(cli_ctx, instance, scope):
-    managed_identity_operator_role = 'Managed Identity Operator'
-    managed_identity_operator_role_id = 'f1a07417-d97a-45cb-824c-7a7467783830'
-
     cluster_identity_object_id = None
     if instance.identity.type.lower() == 'userassigned':
         for identity in instance.identity.user_assigned_identities.values():
@@ -3915,14 +3936,14 @@ def _ensure_managed_identity_operator_permission(cli_ctx, instance, scope):
     for i in assignments_client.list_for_scope(scope=scope, filter='atScope()'):
         if i.scope.lower() != scope.lower():
             continue
-        if not i.role_definition_id.lower().endswith(managed_identity_operator_role_id):
+        if not i.role_definition_id.lower().endswith(CONST_MANAGED_IDENTITY_OPERATOR_ROLE_ID):
             continue
         if i.principal_id.lower() != cluster_identity_object_id.lower():
             continue
         # already assigned
         return
 
-    if not _add_role_assignment(cli_ctx, managed_identity_operator_role, cluster_identity_object_id,
+    if not _add_role_assignment(cli_ctx, CONST_MANAGED_IDENTITY_OPERATOR_ROLE, cluster_identity_object_id,
                                 is_service_principal=False, scope=scope):
         raise CLIError(
             'Could not grant Managed Identity Operator permission for cluster')
@@ -4076,3 +4097,22 @@ def aks_pod_identity_exception_update(cmd, client, resource_group_name, cluster_
 def aks_pod_identity_exception_list(cmd, client, resource_group_name, cluster_name):
     instance = client.get(resource_group_name, cluster_name)
     return _remove_nulls([instance])[0]
+
+
+def _ensure_cluster_identity_permission_on_kubelet_identity(cli_ctx, cluster_identity_object_id, scope):
+    factory = get_auth_management_client(cli_ctx, scope)
+    assignments_client = factory.role_assignments
+
+    for i in assignments_client.list_for_scope(scope=scope, filter='atScope()'):
+        if i.scope.lower() != scope.lower():
+            continue
+        if not i.role_definition_id.lower().endswith(CONST_MANAGED_IDENTITY_OPERATOR_ROLE_ID):
+            continue
+        if i.principal_id.lower() != cluster_identity_object_id.lower():
+            continue
+        # already assigned
+        return
+
+    if not _add_role_assignment(cli_ctx, CONST_MANAGED_IDENTITY_OPERATOR_ROLE, cluster_identity_object_id,
+                                is_service_principal=False, scope=scope):
+        raise CLIError('Could not grant Managed Identity Operator permission to cluster identity at scope {}'.format(scope))