From e62dd0d3ddfd826b3ceaa8dcd583d7826d47d04d Mon Sep 17 00:00:00 2001 From: Jon Huhn Date: Tue, 25 Jun 2024 16:51:03 +0000 Subject: [PATCH] Update Azure cluster-autoscaler e2e cluster template --- ...plate-prow-aks-aso-cluster-autoscaler.yaml | 153 ++++++++++++------ 1 file changed, 105 insertions(+), 48 deletions(-) diff --git a/cluster-autoscaler/cloudprovider/azure/test/templates/cluster-template-prow-aks-aso-cluster-autoscaler.yaml b/cluster-autoscaler/cloudprovider/azure/test/templates/cluster-template-prow-aks-aso-cluster-autoscaler.yaml index 0c1d712dc508..8cf989e9e174 100644 --- a/cluster-autoscaler/cloudprovider/azure/test/templates/cluster-template-prow-aks-aso-cluster-autoscaler.yaml +++ b/cluster-autoscaler/cloudprovider/azure/test/templates/cluster-template-prow-aks-aso-cluster-autoscaler.yaml @@ -19,6 +19,7 @@ metadata: name: ${CLUSTER_NAME} namespace: default spec: + version: ${KUBERNETES_VERSION} resources: - apiVersion: containerservice.azure.com/v1api20231001 kind: ManagedCluster @@ -52,7 +53,54 @@ spec: buildProvenance: ${BUILD_PROVENANCE} creationTimestamp: ${TIMESTAMP} jobName: ${JOB_NAME} - version: ${KUBERNETES_VERSION} + - apiVersion: managedidentity.azure.com/v1api20230131 + kind: UserAssignedIdentity + metadata: + annotations: + serviceoperator.azure.com/credential-from: ${ASO_CREDENTIAL_SECRET_NAME} + name: ${CLUSTER_NAME} + namespace: default + spec: + location: ${AZURE_LOCATION} + operatorSpec: + configMaps: + principalId: + key: principal-id + name: ${CLUSTER_NAME}-identity + owner: + name: ${CLUSTER_NAME} + - apiVersion: managedidentity.azure.com/v1api20230131 + kind: FederatedIdentityCredential + metadata: + annotations: + serviceoperator.azure.com/credential-from: ${ASO_CREDENTIAL_SECRET_NAME} + name: ${CLUSTER_NAME} + namespace: default + spec: + audiences: + - api://AzureADTokenExchange + issuerFromConfig: + key: issuer + name: ${CLUSTER_NAME}-oidc + owner: + name: ${CLUSTER_NAME} + subject: system:serviceaccount:${CLUSTER_AUTOSCALER_NAMESPACE:=default}:${CLUSTER_AUTOSCALER_SERVICEACCOUNT_NAME:=cluster-autoscaler} + - apiVersion: authorization.azure.com/v1api20220401 + kind: RoleAssignment + metadata: + annotations: + serviceoperator.azure.com/credential-from: ${ASO_CREDENTIAL_SECRET_NAME} + name: ${CLUSTER_NAME} + namespace: default + spec: + owner: + armId: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/MC_${CLUSTER_NAME}_${CLUSTER_NAME}_${AZURE_LOCATION} + principalIdFromConfig: + key: principal-id + name: ${CLUSTER_NAME}-identity + roleDefinitionReference: + # Contributor + armId: /subscriptions/${AZURE_SUBSCRIPTION_ID}/providers/Microsoft.Authorization/roleDefinitions/b24988ac-6180-42a0-ab88-20f7382dd24c --- apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 kind: AzureASOManagedCluster @@ -219,53 +267,62 @@ spec: type: VirtualMachineScaleSets vmSize: ${AZURE_AKS_NODE_MACHINE_TYPE:=Standard_D2s_v3} --- -apiVersion: managedidentity.azure.com/v1api20230131 -kind: UserAssignedIdentity -metadata: - annotations: - serviceoperator.azure.com/credential-from: ${ASO_CREDENTIAL_SECRET_NAME} - name: ${CLUSTER_NAME} - namespace: default -spec: - location: ${AZURE_LOCATION} - operatorSpec: - configMaps: - principalId: - key: principal-id - name: ${CLUSTER_NAME}-identity - owner: - name: ${CLUSTER_NAME} ---- -apiVersion: managedidentity.azure.com/v1api20230131 -kind: FederatedIdentityCredential +apiVersion: v1 +kind: Secret metadata: - annotations: - serviceoperator.azure.com/credential-from: ${ASO_CREDENTIAL_SECRET_NAME} - name: ${CLUSTER_NAME} - namespace: default -spec: - audiences: - - api://AzureADTokenExchange - issuerFromConfig: - key: issuer - name: ${CLUSTER_NAME}-oidc - owner: - name: ${CLUSTER_NAME} - subject: system:serviceaccount:${CLUSTER_AUTOSCALER_NAMESPACE:=default}:${CLUSTER_AUTOSCALER_SERVICEACCOUNT_NAME:=cluster-autoscaler} + name: ${ASO_CREDENTIAL_SECRET_NAME} +stringData: + AZURE_SUBSCRIPTION_ID: ${AZURE_SUBSCRIPTION_ID} + AZURE_TENANT_ID: ${AZURE_TENANT_ID} + AZURE_CLIENT_ID: ${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY} + AUTH_MODE: ${ASO_CREDENTIAL_SECRET_MODE:-workloadidentity} --- -apiVersion: authorization.azure.com/v1api20220401 -kind: RoleAssignment +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole metadata: - annotations: - serviceoperator.azure.com/credential-from: ${ASO_CREDENTIAL_SECRET_NAME} - name: ${CLUSTER_NAME} - namespace: default -spec: - owner: - armId: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/MC_${CLUSTER_NAME}_${CLUSTER_NAME}_${AZURE_LOCATION} - principalIdFromConfig: - key: principal-id - name: ${CLUSTER_NAME}-identity - roleDefinitionReference: - # Contributor - armId: /subscriptions/${AZURE_SUBSCRIPTION_ID}/providers/Microsoft.Authorization/roleDefinitions/b24988ac-6180-42a0-ab88-20f7382dd24c + name: capz-${CLUSTER_NAME} + labels: + cluster.x-k8s.io/aggregate-to-capz-manager: "true" +rules: +- apiGroups: + - authorization.azure.com + resources: + - roleassignments + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - authorization.azure.com + resources: + - roleassignments/status + verbs: + - get + - list + - watch +- apiGroups: + - managedidentity.azure.com + resources: + - userassignedidentities + - federatedidentitycredentials + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - managedidentity.azure.com + resources: + - userassignedidentities/status + - federatedidentitycredentials/status + verbs: + - get + - list + - watch