diff --git a/cluster-autoscaler/cloudprovider/azure/test/Makefile b/cluster-autoscaler/cloudprovider/azure/test/Makefile new file mode 100644 index 000000000000..9d5cd99a08b2 --- /dev/null +++ b/cluster-autoscaler/cloudprovider/azure/test/Makefile @@ -0,0 +1,53 @@ +REPO_ROOT:=$(shell git rev-parse --show-toplevel) +CAS_ROOT:=$(REPO_ROOT)/cluster-autoscaler +CAS_CHART:=$(REPO_ROOT)/charts/cluster-autoscaler + +include $(CAS_ROOT)/Makefile + +TOOLS_BIN_DIR := $(abspath bin) + +export PATH := $(TOOLS_BIN_DIR):$(PATH) + +HELM_VER := v3.15.2 +HELM_BIN := helm +HELM := $(TOOLS_BIN_DIR)/$(HELM_BIN)-$(HELM_VER) + +helm: $(HELM) + +$(HELM): + mkdir -p $(TOOLS_BIN_DIR) + rm -f "$(TOOLS_BIN_DIR)/$(HELM_BIN)*" + curl --retry 3 -fsSL -o $(TOOLS_BIN_DIR)/get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 + chmod 700 $(TOOLS_BIN_DIR)/get_helm.sh + USE_SUDO=false HELM_INSTALL_DIR=$(TOOLS_BIN_DIR) DESIRED_VERSION=$(HELM_VER) BINARY_NAME=$(HELM_BIN)-$(HELM_VER) $(TOOLS_BIN_DIR)/get_helm.sh + ln -sf $(HELM) $(TOOLS_BIN_DIR)/$(HELM_BIN) + rm -f $(TOOLS_BIN_DIR)/get_helm.sh + +CLUSTER_AUTOSCALER_NAMESPACE?=default +CLUSTER_AUTOSCALER_SERVICEACCOUNT_NAME?=cluster-autoscaler + +.PHONY: install-e2e +install-e2e: $(HELM) + $(MAKE) -C $(CAS_ROOT) build-arch-$(GOARCH) make-image-arch-$(GOARCH) + docker push $(IMAGE)-$(GOARCH):$(TAG) + $(HELM) install cluster-autoscaler $(CAS_CHART) \ + --namespace $(CLUSTER_AUTOSCALER_NAMESPACE) --create-namespace \ + --set cloudProvider=azure \ + --set azureTenantID=$(AZURE_TENANT_ID) \ + --set azureSubscriptionID=$(AZURE_SUBSCRIPTION_ID) \ + --set azureUseWorkloadIdentityExtension=true \ + --set-string podLabels."azure\.workload\.identity/use"=true \ + --set rbac.serviceAccount.name=$(CLUSTER_AUTOSCALER_SERVICEACCOUNT_NAME) \ + --set rbac.serviceAccount.annotations."azure\.workload\.identity/tenant-id"=$(AZURE_TENANT_ID) \ + --set rbac.serviceAccount.annotations."azure\.workload\.identity/client-id"="$$(KUBECONFIG= kubectl get userassignedidentities -o jsonpath='{.items[0].status.clientId}')" \ + --set autoDiscovery.clusterName="$$(KUBECONFIG= kubectl get cluster -o jsonpath='{.items[0].metadata.name}')" \ + --set azureResourceGroup="$$(KUBECONFIG= kubectl get managedclusters -o jsonpath='{.items[0].status.nodeResourceGroup}')" \ + --set nodeSelector."kubernetes\.io/os"=linux \ + --set image.repository=$(IMAGE)-$(GOARCH) \ + --set image.tag=$(TAG) \ + --set image.pullPolicy=Always \ + --wait + +.PHONY: test-e2e +test-e2e: install-e2e + # TODO diff --git a/cluster-autoscaler/cloudprovider/azure/test/templates/cluster-template-prow-aks-aso-cluster-autoscaler.yaml b/cluster-autoscaler/cloudprovider/azure/test/templates/cluster-template-prow-aks-aso-cluster-autoscaler.yaml new file mode 100644 index 000000000000..0c1d712dc508 --- /dev/null +++ b/cluster-autoscaler/cloudprovider/azure/test/templates/cluster-template-prow-aks-aso-cluster-autoscaler.yaml @@ -0,0 +1,271 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + controlPlaneRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + kind: AzureASOManagedControlPlane + name: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + kind: AzureASOManagedCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 +kind: AzureASOManagedControlPlane +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + resources: + - apiVersion: containerservice.azure.com/v1api20231001 + kind: ManagedCluster + metadata: + annotations: + serviceoperator.azure.com/credential-from: ${ASO_CREDENTIAL_SECRET_NAME} + name: ${CLUSTER_NAME} + spec: + dnsPrefix: ${CLUSTER_NAME} + identity: + type: SystemAssigned + location: ${AZURE_LOCATION} + networkProfile: + networkPlugin: azure + nodeResourceGroup: MC_${CLUSTER_NAME}_${CLUSTER_NAME}_${AZURE_LOCATION} + oidcIssuerProfile: + enabled: true + operatorSpec: + configMaps: + oidcIssuerProfile: + key: issuer + name: ${CLUSTER_NAME}-oidc + owner: + name: ${CLUSTER_NAME} + securityProfile: + workloadIdentity: + enabled: true + servicePrincipalProfile: + clientId: msi + tags: + buildProvenance: ${BUILD_PROVENANCE} + creationTimestamp: ${TIMESTAMP} + jobName: ${JOB_NAME} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 +kind: AzureASOManagedCluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + resources: + - apiVersion: resources.azure.com/v1api20200601 + kind: ResourceGroup + metadata: + annotations: + serviceoperator.azure.com/credential-from: ${ASO_CREDENTIAL_SECRET_NAME} + name: ${CLUSTER_NAME} + spec: + location: ${AZURE_LOCATION} + tags: + buildProvenance: ${BUILD_PROVENANCE} + creationTimestamp: ${TIMESTAMP} + jobName: ${JOB_NAME} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachinePool +metadata: + annotations: + cluster.x-k8s.io/replicas-managed-by: cluster-autoscaler + name: ${CLUSTER_NAME}-pool0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT:=2} + template: + metadata: {} + spec: + bootstrap: + dataSecretName: "" + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + kind: AzureASOManagedMachinePool + name: ${CLUSTER_NAME}-pool0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 +kind: AzureASOManagedMachinePool +metadata: + name: ${CLUSTER_NAME}-pool0 + namespace: default +spec: + resources: + - apiVersion: containerservice.azure.com/v1api20231001 + kind: ManagedClustersAgentPool + metadata: + annotations: + serviceoperator.azure.com/credential-from: ${ASO_CREDENTIAL_SECRET_NAME} + name: ${CLUSTER_NAME}-pool0 + spec: + azureName: pool0 + mode: System + owner: + name: ${CLUSTER_NAME} + tags: + cluster-autoscaler-enabled: "true" + cluster-autoscaler-name: ${CLUSTER_NAME} + max: "5" + min: "1" + type: VirtualMachineScaleSets + vmSize: ${AZURE_AKS_NODE_MACHINE_TYPE:=Standard_D2s_v3} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachinePool +metadata: + annotations: + cluster.x-k8s.io/replicas-managed-by: cluster-autoscaler + name: ${CLUSTER_NAME}-pool1 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT:=2} + template: + metadata: {} + spec: + bootstrap: + dataSecretName: "" + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + kind: AzureASOManagedMachinePool + name: ${CLUSTER_NAME}-pool1 + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 +kind: AzureASOManagedMachinePool +metadata: + name: ${CLUSTER_NAME}-pool1 + namespace: default +spec: + resources: + - apiVersion: containerservice.azure.com/v1api20231001 + kind: ManagedClustersAgentPool + metadata: + annotations: + serviceoperator.azure.com/credential-from: ${ASO_CREDENTIAL_SECRET_NAME} + name: ${CLUSTER_NAME}-pool1 + spec: + azureName: pool1 + mode: User + owner: + name: ${CLUSTER_NAME} + tags: + cluster-autoscaler-enabled: "true" + cluster-autoscaler-name: ${CLUSTER_NAME} + max: "5" + min: "1" + type: VirtualMachineScaleSets + vmSize: ${AZURE_AKS_NODE_MACHINE_TYPE:=Standard_D2s_v3} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachinePool +metadata: + annotations: + cluster.x-k8s.io/replicas-managed-by: cluster-autoscaler + name: ${CLUSTER_NAME}-pool2 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: 1 + template: + metadata: {} + spec: + bootstrap: + dataSecretName: "" + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + kind: AzureASOManagedMachinePool + name: ${CLUSTER_NAME}-pool2 + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 +kind: AzureASOManagedMachinePool +metadata: + name: ${CLUSTER_NAME}-pool2 + namespace: default +spec: + resources: + - apiVersion: containerservice.azure.com/v1api20231001 + kind: ManagedClustersAgentPool + metadata: + annotations: + serviceoperator.azure.com/credential-from: ${ASO_CREDENTIAL_SECRET_NAME} + name: ${CLUSTER_NAME}-pool2 + spec: + azureName: pool2 + mode: User + osType: Windows + owner: + name: ${CLUSTER_NAME} + tags: + cluster-autoscaler-enabled: "true" + cluster-autoscaler-name: ${CLUSTER_NAME} + max: "5" + min: "1" + type: VirtualMachineScaleSets + vmSize: ${AZURE_AKS_NODE_MACHINE_TYPE:=Standard_D2s_v3} +--- +apiVersion: managedidentity.azure.com/v1api20230131 +kind: UserAssignedIdentity +metadata: + annotations: + serviceoperator.azure.com/credential-from: ${ASO_CREDENTIAL_SECRET_NAME} + name: ${CLUSTER_NAME} + namespace: default +spec: + location: ${AZURE_LOCATION} + operatorSpec: + configMaps: + principalId: + key: principal-id + name: ${CLUSTER_NAME}-identity + owner: + name: ${CLUSTER_NAME} +--- +apiVersion: managedidentity.azure.com/v1api20230131 +kind: FederatedIdentityCredential +metadata: + annotations: + serviceoperator.azure.com/credential-from: ${ASO_CREDENTIAL_SECRET_NAME} + name: ${CLUSTER_NAME} + namespace: default +spec: + audiences: + - api://AzureADTokenExchange + issuerFromConfig: + key: issuer + name: ${CLUSTER_NAME}-oidc + owner: + name: ${CLUSTER_NAME} + subject: system:serviceaccount:${CLUSTER_AUTOSCALER_NAMESPACE:=default}:${CLUSTER_AUTOSCALER_SERVICEACCOUNT_NAME:=cluster-autoscaler} +--- +apiVersion: authorization.azure.com/v1api20220401 +kind: RoleAssignment +metadata: + annotations: + serviceoperator.azure.com/credential-from: ${ASO_CREDENTIAL_SECRET_NAME} + name: ${CLUSTER_NAME} + namespace: default +spec: + owner: + armId: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/MC_${CLUSTER_NAME}_${CLUSTER_NAME}_${AZURE_LOCATION} + principalIdFromConfig: + key: principal-id + name: ${CLUSTER_NAME}-identity + roleDefinitionReference: + # Contributor + armId: /subscriptions/${AZURE_SUBSCRIPTION_ID}/providers/Microsoft.Authorization/roleDefinitions/b24988ac-6180-42a0-ab88-20f7382dd24c