Merge pull request #6975 from nojnhuh/azure-e2e-test

Update Azure cluster-autoscaler e2e cluster template
kubernetes · Jun 27, 2024 · bab68c4 · bab68c4
2 parents 3d48578 + e62dd0d
commit bab68c4
Showing 1 changed file with 105 additions and 48 deletions.
diff --git a/.../cloudprovider/azure/test/templates/cluster-template-prow-aks-aso-cluster-autoscaler.yaml b/.../cloudprovider/azure/test/templates/cluster-template-prow-aks-aso-cluster-autoscaler.yaml
@@ -19,6 +19,7 @@ metadata:
   name: ${CLUSTER_NAME}
   namespace: default
 spec:
+  version: ${KUBERNETES_VERSION}
   resources:
   - apiVersion: containerservice.azure.com/v1api20231001
     kind: ManagedCluster
@@ -52,7 +53,54 @@ spec:
         buildProvenance: ${BUILD_PROVENANCE}
         creationTimestamp: ${TIMESTAMP}
         jobName: ${JOB_NAME}
-  version: ${KUBERNETES_VERSION}
+  - apiVersion: managedidentity.azure.com/v1api20230131
+    kind: UserAssignedIdentity
+    metadata:
+      annotations:
+        serviceoperator.azure.com/credential-from: ${ASO_CREDENTIAL_SECRET_NAME}
+      name: ${CLUSTER_NAME}
+      namespace: default
+    spec:
+      location: ${AZURE_LOCATION}
+      operatorSpec:
+        configMaps:
+          principalId:
+            key: principal-id
+            name: ${CLUSTER_NAME}-identity
+      owner:
+        name: ${CLUSTER_NAME}
+  - apiVersion: managedidentity.azure.com/v1api20230131
+    kind: FederatedIdentityCredential
+    metadata:
+      annotations:
+        serviceoperator.azure.com/credential-from: ${ASO_CREDENTIAL_SECRET_NAME}
+      name: ${CLUSTER_NAME}
+      namespace: default
+    spec:
+      audiences:
+      - api://AzureADTokenExchange
+      issuerFromConfig:
+        key: issuer
+        name: ${CLUSTER_NAME}-oidc
+      owner:
+        name: ${CLUSTER_NAME}
+      subject: system:serviceaccount:${CLUSTER_AUTOSCALER_NAMESPACE:=default}:${CLUSTER_AUTOSCALER_SERVICEACCOUNT_NAME:=cluster-autoscaler}
+  - apiVersion: authorization.azure.com/v1api20220401
+    kind: RoleAssignment
+    metadata:
+      annotations:
+        serviceoperator.azure.com/credential-from: ${ASO_CREDENTIAL_SECRET_NAME}
+      name: ${CLUSTER_NAME}
+      namespace: default
+    spec:
+      owner:
+        armId: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/MC_${CLUSTER_NAME}_${CLUSTER_NAME}_${AZURE_LOCATION}
+      principalIdFromConfig:
+        key: principal-id
+        name: ${CLUSTER_NAME}-identity
+      roleDefinitionReference:
+        # Contributor
+        armId: /subscriptions/${AZURE_SUBSCRIPTION_ID}/providers/Microsoft.Authorization/roleDefinitions/b24988ac-6180-42a0-ab88-20f7382dd24c
 ---
 apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
 kind: AzureASOManagedCluster
@@ -219,53 +267,62 @@ spec:
       type: VirtualMachineScaleSets
       vmSize: ${AZURE_AKS_NODE_MACHINE_TYPE:=Standard_D2s_v3}
 ---
-apiVersion: managedidentity.azure.com/v1api20230131
-kind: UserAssignedIdentity
-metadata:
-  annotations:
-    serviceoperator.azure.com/credential-from: ${ASO_CREDENTIAL_SECRET_NAME}
-  name: ${CLUSTER_NAME}
-  namespace: default
-spec:
-  location: ${AZURE_LOCATION}
-  operatorSpec:
-    configMaps:
-      principalId:
-        key: principal-id
-        name: ${CLUSTER_NAME}-identity
-  owner:
-    name: ${CLUSTER_NAME}
----
-apiVersion: managedidentity.azure.com/v1api20230131
-kind: FederatedIdentityCredential
+apiVersion: v1
+kind: Secret
 metadata:
-  annotations:
-    serviceoperator.azure.com/credential-from: ${ASO_CREDENTIAL_SECRET_NAME}
-  name: ${CLUSTER_NAME}
-  namespace: default
-spec:
-  audiences:
-  - api://AzureADTokenExchange
-  issuerFromConfig:
-    key: issuer
-    name: ${CLUSTER_NAME}-oidc
-  owner:
-    name: ${CLUSTER_NAME}
-  subject: system:serviceaccount:${CLUSTER_AUTOSCALER_NAMESPACE:=default}:${CLUSTER_AUTOSCALER_SERVICEACCOUNT_NAME:=cluster-autoscaler}
+  name: ${ASO_CREDENTIAL_SECRET_NAME}
+stringData:
+  AZURE_SUBSCRIPTION_ID: ${AZURE_SUBSCRIPTION_ID}
+  AZURE_TENANT_ID:       ${AZURE_TENANT_ID}
+  AZURE_CLIENT_ID:       ${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY}
+  AUTH_MODE:             ${ASO_CREDENTIAL_SECRET_MODE:-workloadidentity}
 ---
-apiVersion: authorization.azure.com/v1api20220401
-kind: RoleAssignment
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
 metadata:
-  annotations:
-    serviceoperator.azure.com/credential-from: ${ASO_CREDENTIAL_SECRET_NAME}
-  name: ${CLUSTER_NAME}
-  namespace: default
-spec:
-  owner:
-    armId: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/MC_${CLUSTER_NAME}_${CLUSTER_NAME}_${AZURE_LOCATION}
-  principalIdFromConfig:
-    key: principal-id
-    name: ${CLUSTER_NAME}-identity
-  roleDefinitionReference:
-    # Contributor
-    armId: /subscriptions/${AZURE_SUBSCRIPTION_ID}/providers/Microsoft.Authorization/roleDefinitions/b24988ac-6180-42a0-ab88-20f7382dd24c
+  name: capz-${CLUSTER_NAME}
+  labels:
+    cluster.x-k8s.io/aggregate-to-capz-manager: "true"
+rules:
+- apiGroups:
+  - authorization.azure.com
+  resources:
+  - roleassignments
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - authorization.azure.com
+  resources:
+  - roleassignments/status
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - managedidentity.azure.com
+  resources:
+  - userassignedidentities
+  - federatedidentitycredentials
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - managedidentity.azure.com
+  resources:
+  - userassignedidentities/status
+  - federatedidentitycredentials/status
+  verbs:
+  - get
+  - list
+  - watch