Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add initial setup for cluster-autoscaler Azure e2e tests #6969

Merged
merged 1 commit into from
Jun 24, 2024
+324 −0
Merged

Add initial setup for cluster-autoscaler Azure e2e tests #6969

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 53 additions & 0 deletions cluster-autoscaler/cloudprovider/azure/test/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
REPO_ROOT:=$(shell git rev-parse --show-toplevel)
CAS_ROOT:=$(REPO_ROOT)/cluster-autoscaler
CAS_CHART:=$(REPO_ROOT)/charts/cluster-autoscaler

include $(CAS_ROOT)/Makefile

TOOLS_BIN_DIR := $(abspath bin)

export PATH := $(TOOLS_BIN_DIR):$(PATH)

HELM_VER := v3.15.2
HELM_BIN := helm
HELM := $(TOOLS_BIN_DIR)/$(HELM_BIN)-$(HELM_VER)

helm: $(HELM)

$(HELM):
mkdir -p $(TOOLS_BIN_DIR)
rm -f "$(TOOLS_BIN_DIR)/$(HELM_BIN)*"
curl --retry 3 -fsSL -o $(TOOLS_BIN_DIR)/get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
chmod 700 $(TOOLS_BIN_DIR)/get_helm.sh
USE_SUDO=false HELM_INSTALL_DIR=$(TOOLS_BIN_DIR) DESIRED_VERSION=$(HELM_VER) BINARY_NAME=$(HELM_BIN)-$(HELM_VER) $(TOOLS_BIN_DIR)/get_helm.sh
ln -sf $(HELM) $(TOOLS_BIN_DIR)/$(HELM_BIN)
rm -f $(TOOLS_BIN_DIR)/get_helm.sh

CLUSTER_AUTOSCALER_NAMESPACE?=default
CLUSTER_AUTOSCALER_SERVICEACCOUNT_NAME?=cluster-autoscaler

.PHONY: install-e2e
install-e2e: $(HELM)
$(MAKE) -C $(CAS_ROOT) build-arch-$(GOARCH) make-image-arch-$(GOARCH)
docker push $(IMAGE)-$(GOARCH):$(TAG)
$(HELM) install cluster-autoscaler $(CAS_CHART) \
--namespace $(CLUSTER_AUTOSCALER_NAMESPACE) --create-namespace \
--set cloudProvider=azure \
--set azureTenantID=$(AZURE_TENANT_ID) \
--set azureSubscriptionID=$(AZURE_SUBSCRIPTION_ID) \
--set azureUseWorkloadIdentityExtension=true \
--set-string podLabels."azure\.workload\.identity/use"=true \
--set rbac.serviceAccount.name=$(CLUSTER_AUTOSCALER_SERVICEACCOUNT_NAME) \
--set rbac.serviceAccount.annotations."azure\.workload\.identity/tenant-id"=$(AZURE_TENANT_ID) \
--set rbac.serviceAccount.annotations."azure\.workload\.identity/client-id"="$$(KUBECONFIG= kubectl get userassignedidentities -o jsonpath='{.items[0].status.clientId}')" \
--set autoDiscovery.clusterName="$$(KUBECONFIG= kubectl get cluster -o jsonpath='{.items[0].metadata.name}')" \
--set azureResourceGroup="$$(KUBECONFIG= kubectl get managedclusters -o jsonpath='{.items[0].status.nodeResourceGroup}')" \
--set nodeSelector."kubernetes\.io/os"=linux \
--set image.repository=$(IMAGE)-$(GOARCH) \
--set image.tag=$(TAG) \
--set image.pullPolicy=Always \
--wait

.PHONY: test-e2e
test-e2e: install-e2e
# TODO
271 changes: 271 additions & 0 deletions .../cloudprovider/azure/test/templates/cluster-template-prow-aks-aso-cluster-autoscaler.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,271 @@
apiVersion: cluster.x-k8s.io/v1beta1
kind: Cluster
metadata:
name: ${CLUSTER_NAME}
namespace: default
spec:
controlPlaneRef:
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
kind: AzureASOManagedControlPlane
name: ${CLUSTER_NAME}
infrastructureRef:
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
kind: AzureASOManagedCluster
name: ${CLUSTER_NAME}
---
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
kind: AzureASOManagedControlPlane
metadata:
name: ${CLUSTER_NAME}
namespace: default
spec:
resources:
- apiVersion: containerservice.azure.com/v1api20231001
kind: ManagedCluster
metadata:
annotations:
serviceoperator.azure.com/credential-from: ${ASO_CREDENTIAL_SECRET_NAME}
name: ${CLUSTER_NAME}
spec:
dnsPrefix: ${CLUSTER_NAME}
identity:
type: SystemAssigned
location: ${AZURE_LOCATION}
networkProfile:
networkPlugin: azure
nodeResourceGroup: MC_${CLUSTER_NAME}_${CLUSTER_NAME}_${AZURE_LOCATION}
oidcIssuerProfile:
enabled: true
operatorSpec:
configMaps:
oidcIssuerProfile:
key: issuer
name: ${CLUSTER_NAME}-oidc
owner:
name: ${CLUSTER_NAME}
securityProfile:
workloadIdentity:
enabled: true
servicePrincipalProfile:
clientId: msi
tags:
buildProvenance: ${BUILD_PROVENANCE}
creationTimestamp: ${TIMESTAMP}
jobName: ${JOB_NAME}
version: ${KUBERNETES_VERSION}
---
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
kind: AzureASOManagedCluster
metadata:
name: ${CLUSTER_NAME}
namespace: default
spec:
resources:
- apiVersion: resources.azure.com/v1api20200601
kind: ResourceGroup
metadata:
annotations:
serviceoperator.azure.com/credential-from: ${ASO_CREDENTIAL_SECRET_NAME}
name: ${CLUSTER_NAME}
spec:
location: ${AZURE_LOCATION}
tags:
buildProvenance: ${BUILD_PROVENANCE}
creationTimestamp: ${TIMESTAMP}
jobName: ${JOB_NAME}
---
apiVersion: cluster.x-k8s.io/v1beta1
kind: MachinePool
metadata:
annotations:
cluster.x-k8s.io/replicas-managed-by: cluster-autoscaler
name: ${CLUSTER_NAME}-pool0
namespace: default
spec:
clusterName: ${CLUSTER_NAME}
replicas: ${WORKER_MACHINE_COUNT:=2}
template:
metadata: {}
spec:
bootstrap:
dataSecretName: ""
clusterName: ${CLUSTER_NAME}
infrastructureRef:
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
kind: AzureASOManagedMachinePool
name: ${CLUSTER_NAME}-pool0
version: ${KUBERNETES_VERSION}
---
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
kind: AzureASOManagedMachinePool
metadata:
name: ${CLUSTER_NAME}-pool0
namespace: default
spec:
resources:
- apiVersion: containerservice.azure.com/v1api20231001
kind: ManagedClustersAgentPool
metadata:
annotations:
serviceoperator.azure.com/credential-from: ${ASO_CREDENTIAL_SECRET_NAME}
name: ${CLUSTER_NAME}-pool0
spec:
azureName: pool0
mode: System
owner:
name: ${CLUSTER_NAME}
tags:
cluster-autoscaler-enabled: "true"
cluster-autoscaler-name: ${CLUSTER_NAME}
max: "5"
min: "1"
type: VirtualMachineScaleSets
vmSize: ${AZURE_AKS_NODE_MACHINE_TYPE:=Standard_D2s_v3}
---
apiVersion: cluster.x-k8s.io/v1beta1
kind: MachinePool
metadata:
annotations:
cluster.x-k8s.io/replicas-managed-by: cluster-autoscaler
name: ${CLUSTER_NAME}-pool1
namespace: default
spec:
clusterName: ${CLUSTER_NAME}
replicas: ${WORKER_MACHINE_COUNT:=2}
template:
metadata: {}
spec:
bootstrap:
dataSecretName: ""
clusterName: ${CLUSTER_NAME}
infrastructureRef:
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
kind: AzureASOManagedMachinePool
name: ${CLUSTER_NAME}-pool1
version: ${KUBERNETES_VERSION}
---
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
kind: AzureASOManagedMachinePool
metadata:
name: ${CLUSTER_NAME}-pool1
namespace: default
spec:
resources:
- apiVersion: containerservice.azure.com/v1api20231001
kind: ManagedClustersAgentPool
metadata:
annotations:
serviceoperator.azure.com/credential-from: ${ASO_CREDENTIAL_SECRET_NAME}
name: ${CLUSTER_NAME}-pool1
spec:
azureName: pool1
mode: User
owner:
name: ${CLUSTER_NAME}
tags:
cluster-autoscaler-enabled: "true"
cluster-autoscaler-name: ${CLUSTER_NAME}
max: "5"
min: "1"
type: VirtualMachineScaleSets
vmSize: ${AZURE_AKS_NODE_MACHINE_TYPE:=Standard_D2s_v3}
---
apiVersion: cluster.x-k8s.io/v1beta1
kind: MachinePool
metadata:
annotations:
cluster.x-k8s.io/replicas-managed-by: cluster-autoscaler
name: ${CLUSTER_NAME}-pool2
namespace: default
spec:
clusterName: ${CLUSTER_NAME}
replicas: 1
template:
metadata: {}
spec:
bootstrap:
dataSecretName: ""
clusterName: ${CLUSTER_NAME}
infrastructureRef:
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
kind: AzureASOManagedMachinePool
name: ${CLUSTER_NAME}-pool2
version: ${KUBERNETES_VERSION}
---
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
kind: AzureASOManagedMachinePool
metadata:
name: ${CLUSTER_NAME}-pool2
namespace: default
spec:
resources:
- apiVersion: containerservice.azure.com/v1api20231001
kind: ManagedClustersAgentPool
metadata:
annotations:
serviceoperator.azure.com/credential-from: ${ASO_CREDENTIAL_SECRET_NAME}
name: ${CLUSTER_NAME}-pool2
spec:
azureName: pool2
mode: User
osType: Windows
owner:
name: ${CLUSTER_NAME}
tags:
cluster-autoscaler-enabled: "true"
cluster-autoscaler-name: ${CLUSTER_NAME}
max: "5"
min: "1"
type: VirtualMachineScaleSets
vmSize: ${AZURE_AKS_NODE_MACHINE_TYPE:=Standard_D2s_v3}
---
apiVersion: managedidentity.azure.com/v1api20230131
kind: UserAssignedIdentity
metadata:
annotations:
serviceoperator.azure.com/credential-from: ${ASO_CREDENTIAL_SECRET_NAME}
name: ${CLUSTER_NAME}
namespace: default
spec:
location: ${AZURE_LOCATION}
operatorSpec:
configMaps:
principalId:
key: principal-id
name: ${CLUSTER_NAME}-identity
owner:
name: ${CLUSTER_NAME}
---
apiVersion: managedidentity.azure.com/v1api20230131
kind: FederatedIdentityCredential
metadata:
annotations:
serviceoperator.azure.com/credential-from: ${ASO_CREDENTIAL_SECRET_NAME}
name: ${CLUSTER_NAME}
namespace: default
spec:
audiences:
- api://AzureADTokenExchange
issuerFromConfig:
key: issuer
name: ${CLUSTER_NAME}-oidc
owner:
name: ${CLUSTER_NAME}
subject: system:serviceaccount:${CLUSTER_AUTOSCALER_NAMESPACE:=default}:${CLUSTER_AUTOSCALER_SERVICEACCOUNT_NAME:=cluster-autoscaler}
---
apiVersion: authorization.azure.com/v1api20220401
kind: RoleAssignment
metadata:
annotations:
serviceoperator.azure.com/credential-from: ${ASO_CREDENTIAL_SECRET_NAME}
name: ${CLUSTER_NAME}
namespace: default
spec:
owner:
armId: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/MC_${CLUSTER_NAME}_${CLUSTER_NAME}_${AZURE_LOCATION}
principalIdFromConfig:
key: principal-id
name: ${CLUSTER_NAME}-identity
roleDefinitionReference:
# Contributor
armId: /subscriptions/${AZURE_SUBSCRIPTION_ID}/providers/Microsoft.Authorization/roleDefinitions/b24988ac-6180-42a0-ab88-20f7382dd24c
Loading