Skip to content

Commit

Permalink
Merge pull request kubernetes#6994 from nojnhuh/azure-e2e-test
Browse files Browse the repository at this point in the history
cluster-autoscaler Azure e2e: move Helm to ginkgo
  • Loading branch information
k8s-ci-robot authored Jul 3, 2024
2 parents 4d05113 + 3f18f06 commit bb5d064
Show file tree
Hide file tree
Showing 5 changed files with 515 additions and 71 deletions.
57 changes: 10 additions & 47 deletions cluster-autoscaler/cloudprovider/azure/test/Makefile
Original file line number Diff line number Diff line change
@@ -1,62 +1,25 @@
REPO_ROOT:=$(shell git rev-parse --show-toplevel)
CAS_ROOT:=$(REPO_ROOT)/cluster-autoscaler
CAS_CHART:=$(REPO_ROOT)/charts/cluster-autoscaler

include $(CAS_ROOT)/Makefile

TOOLS_BIN_DIR := $(abspath bin)

export PATH := $(TOOLS_BIN_DIR):$(PATH)

HELM_VER := v3.15.2
HELM_BIN := helm
HELM := $(TOOLS_BIN_DIR)/$(HELM_BIN)-$(HELM_VER)

helm: $(HELM)

$(HELM):
mkdir -p $(TOOLS_BIN_DIR)
rm -f "$(TOOLS_BIN_DIR)/$(HELM_BIN)*"
curl --retry 3 -fsSL -o $(TOOLS_BIN_DIR)/get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
chmod 700 $(TOOLS_BIN_DIR)/get_helm.sh
USE_SUDO=false HELM_INSTALL_DIR=$(TOOLS_BIN_DIR) DESIRED_VERSION=$(HELM_VER) BINARY_NAME=$(HELM_BIN)-$(HELM_VER) $(TOOLS_BIN_DIR)/get_helm.sh
ln -sf $(HELM) $(TOOLS_BIN_DIR)/$(HELM_BIN)
rm -f $(TOOLS_BIN_DIR)/get_helm.sh

CLUSTER_AUTOSCALER_NAMESPACE?=default
CLUSTER_AUTOSCALER_SERVICEACCOUNT_NAME?=cluster-autoscaler

.PHONY: install-e2e
install-e2e: $(HELM)
.PHONY: build-e2e
build-e2e:
$(MAKE) -C $(CAS_ROOT) build-arch-$(GOARCH) make-image-arch-$(GOARCH)
docker push $(IMAGE)-$(GOARCH):$(TAG)
$(HELM) upgrade --install cluster-autoscaler $(CAS_CHART) \
--namespace $(CLUSTER_AUTOSCALER_NAMESPACE) --create-namespace \
--set cloudProvider=azure \
--set azureTenantID=$(AZURE_TENANT_ID) \
--set azureSubscriptionID=$(AZURE_SUBSCRIPTION_ID) \
--set azureUseWorkloadIdentityExtension=true \
--set-string podLabels."azure\.workload\.identity/use"=true \
--set rbac.serviceAccount.name=$(CLUSTER_AUTOSCALER_SERVICEACCOUNT_NAME) \
--set rbac.serviceAccount.annotations."azure\.workload\.identity/tenant-id"=$(AZURE_TENANT_ID) \
--set rbac.serviceAccount.annotations."azure\.workload\.identity/client-id"="$$(KUBECONFIG= kubectl get userassignedidentities -o jsonpath='{.items[0].status.clientId}')" \
--set autoDiscovery.clusterName="$$(KUBECONFIG= kubectl get cluster -o jsonpath='{.items[0].metadata.name}')" \
--set azureResourceGroup="$$(KUBECONFIG= kubectl get managedclusters -o jsonpath='{.items[0].status.nodeResourceGroup}')" \
--set nodeSelector."kubernetes\.io/os"=linux \
--set image.repository=$(IMAGE)-$(GOARCH) \
--set image.tag=$(TAG) \
--set image.pullPolicy=Always \
--set extraArgs.scale-down-delay-after-add=10s \
--set extraArgs.scale-down-unneeded-time=10s \
--set extraArgs.scale-down-candidates-pool-ratio=1.0 \
--set extraArgs.unremovable-node-recheck-timeout=10s \
--set extraArgs.skip-nodes-with-system-pods=false \
--set extraArgs.skip-nodes-with-local-storage=false \
--wait

ARTIFACTS?=_artifacts

.PHONY: test-e2e
test-e2e: install-e2e
test-e2e: build-e2e
go run github.com/onsi/ginkgo/v2/ginkgo -v --trace --output-dir "$(ARTIFACTS)" --junit-report="junit.e2e_suite.1.xml" e2e -- \
-resource-group="$$(KUBECONFIG= kubectl get managedclusters -o jsonpath='{.items[0].status.nodeResourceGroup}')"
-resource-group="$$(KUBECONFIG= kubectl get managedclusters -o jsonpath='{.items[0].status.nodeResourceGroup}')" \
-cluster-name="$$(KUBECONFIG= kubectl get cluster -o jsonpath='{.items[0].metadata.name}')" \
-client-id="$$(KUBECONFIG= kubectl get userassignedidentities -o jsonpath='{.items[0].status.clientId}')" \
-cas-namespace="$(CLUSTER_AUTOSCALER_NAMESPACE)" \
-cas-serviceaccount-name="$(CLUSTER_AUTOSCALER_SERVICEACCOUNT_NAME)" \
-cas-image-repository="$(IMAGE)-$(GOARCH)" \
-cas-image-tag="$(TAG)"
11 changes: 11 additions & 0 deletions cluster-autoscaler/cloudprovider/azure/test/e2e/azure_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,17 @@ var _ = Describe("Azure Provider", func() {
})

It("scales up AKS node pools when pending Pods exist", func() {
ensureHelmValues(map[string]interface{}{
"extraArgs": map[string]interface{}{
"scale-down-delay-after-add": "10s",
"scale-down-unneeded-time": "10s",
"scale-down-candidates-pool-ratio": "1.0",
"unremovable-node-recheck-timeout": "10s",
"skip-nodes-with-system-pods": "false",
"skip-nodes-with-local-storage": "false",
},
})

nodes := &corev1.NodeList{}
Expect(k8s.List(ctx, nodes)).To(Succeed())
nodeCountBefore := len(nodes.Items)
Expand Down
101 changes: 94 additions & 7 deletions cluster-autoscaler/cloudprovider/azure/test/e2e/e2e_suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,28 +18,52 @@ package e2e_test

import (
"context"
"errors"
"flag"
"fmt"
"os"
"testing"
"time"

"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"helm.sh/helm/v3/pkg/action"
"helm.sh/helm/v3/pkg/chart/loader"
"helm.sh/helm/v3/pkg/cli"
"helm.sh/helm/v3/pkg/storage/driver"
corev1 "k8s.io/api/core/v1"
"k8s.io/cli-runtime/pkg/genericclioptions"
"sigs.k8s.io/controller-runtime/pkg/client"
)

const (
casReleaseName = "cluster-autoscaler"
)

var (
ctx = context.Background()
vmss *armcompute.VirtualMachineScaleSetsClient
k8s client.Client
resourceGroup string
ctx = context.Background()
vmss *armcompute.VirtualMachineScaleSetsClient
k8s client.Client
helmEnv = cli.New()

resourceGroup string
clusterName string
clientID string
casNamespace string
casServiceAccountName string
casImageRepository string
casImageTag string
)

func init() {
flag.StringVar(&resourceGroup, "resource-group", "", "resource group containing cluster-autoscaler-managed resources")
flag.StringVar(&clusterName, "cluster-name", "", "Cluster API Cluster name for the cluster to be managed by cluster-autoscaler")
flag.StringVar(&clientID, "client-id", "", "Azure client ID to be used by cluster-autoscaler")
flag.StringVar(&casNamespace, "cas-namespace", "", "Namespace in which to install cluster-autoscaler")
flag.StringVar(&casServiceAccountName, "cas-serviceaccount-name", "", "Name of the ServiceAccount to be used by cluster-autoscaler")
flag.StringVar(&casImageRepository, "cas-image-repository", "", "Repository of the container image for cluster-autoscaler")
flag.StringVar(&casImageTag, "cas-image-tag", "", "Tag of the container image for cluster-autoscaler")
}

func TestE2E(t *testing.T) {
Expand All @@ -53,11 +77,41 @@ var _ = BeforeSuite(func() {
vmss, err = armcompute.NewVirtualMachineScaleSetsClient(os.Getenv("AZURE_SUBSCRIPTION_ID"), azCred, nil)
Expect(err).NotTo(HaveOccurred())

k8sConfig := genericclioptions.NewConfigFlags(false)
restConfig, err := k8sConfig.ToRESTConfig()
restConfig, err := helmEnv.RESTClientGetter().ToRESTConfig()
Expect(err).NotTo(HaveOccurred())
k8s, err = client.New(restConfig, client.Options{})
Expect(err).NotTo(HaveOccurred())

ensureHelmValues(map[string]interface{}{
"cloudProvider": "azure",
"azureTenantID": os.Getenv("AZURE_TENANT_ID"),
"azureSubscriptionID": os.Getenv("AZURE_SUBSCRIPTION_ID"),
"azureUseWorkloadIdentityExtension": true,
"azureResourceGroup": resourceGroup,
"podLabels": map[string]interface{}{
"azure.workload.identity/use": "true",
},
"rbac": map[string]interface{}{
"serviceAccount": map[string]interface{}{
"name": casServiceAccountName,
"annotations": map[string]interface{}{
"azure.workload.identity/tenant-id": os.Getenv("AZURE_TENANT_ID"),
"azure.workload.identity/client-id": clientID,
},
},
},
"autoDiscovery": map[string]interface{}{
"clusterName": clusterName,
},
"nodeSelector": map[string]interface{}{
"kubernetes.io/os": "linux",
},
"image": map[string]interface{}{
"repository": casImageRepository,
"tag": casImageTag,
"pullPolicy": "Always",
},
})
})

func allVMSSStable(g Gomega) {
Expand Down Expand Up @@ -86,3 +140,36 @@ func allVMSSStable(g Gomega) {
})),
))
}

func ensureHelmValues(values map[string]interface{}) {
helmCfg := new(action.Configuration)
Expect(helmCfg.Init(helmEnv.RESTClientGetter(), casNamespace, "secret", func(format string, v ...interface{}) {
GinkgoLogr.Info(fmt.Sprintf(format, v...))
})).To(Succeed())

chart, err := loader.Load("../../../../../charts/cluster-autoscaler")
Expect(err).NotTo(HaveOccurred())

get := action.NewGet(helmCfg)
_, err = get.Run(casReleaseName)
if errors.Is(err, driver.ErrReleaseNotFound) {
install := action.NewInstall(helmCfg)
install.Timeout = 5 * time.Minute
install.Wait = true
install.CreateNamespace = true
install.ReleaseName = casReleaseName
install.Namespace = casNamespace
_, err := install.Run(chart, values)
Expect(err).NotTo(HaveOccurred())
return
} else {
Expect(err).NotTo(HaveOccurred())
}

upgrade := action.NewUpgrade(helmCfg)
upgrade.Timeout = 5 * time.Minute
upgrade.Wait = true
upgrade.ReuseValues = true
_, err = upgrade.Run(casReleaseName, chart, values)
Expect(err).NotTo(HaveOccurred())
}
87 changes: 81 additions & 6 deletions cluster-autoscaler/cloudprovider/azure/test/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -7,28 +7,56 @@ require (
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0
github.com/onsi/ginkgo/v2 v2.19.0
github.com/onsi/gomega v1.33.1
helm.sh/helm/v3 v3.15.2
k8s.io/api v0.30.2
k8s.io/apimachinery v0.30.2
k8s.io/cli-runtime v0.30.2
k8s.io/utils v0.0.0-20230726121419-3b25d923346b
sigs.k8s.io/controller-runtime v0.18.4
)

require (
github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0 // indirect
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
github.com/BurntSushi/toml v1.3.2 // indirect
github.com/MakeNowJust/heredoc v1.0.0 // indirect
github.com/Masterminds/goutils v1.1.1 // indirect
github.com/Masterminds/semver/v3 v3.2.1 // indirect
github.com/Masterminds/sprig/v3 v3.2.3 // indirect
github.com/Masterminds/squirrel v1.5.4 // indirect
github.com/Microsoft/hcsshim v0.11.4 // indirect
github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/chai2010/gettext-go v1.0.2 // indirect
github.com/containerd/containerd v1.7.12 // indirect
github.com/containerd/log v0.1.0 // indirect
github.com/cyphar/filepath-securejoin v0.2.4 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/distribution/reference v0.5.0 // indirect
github.com/docker/cli v25.0.1+incompatible // indirect
github.com/docker/distribution v2.8.3+incompatible // indirect
github.com/docker/docker v25.0.5+incompatible // indirect
github.com/docker/docker-credential-helpers v0.7.0 // indirect
github.com/docker/go-connections v0.5.0 // indirect
github.com/docker/go-metrics v0.0.1 // indirect
github.com/emicklei/go-restful/v3 v3.11.0 // indirect
github.com/evanphx/json-patch v4.12.0+incompatible // indirect
github.com/evanphx/json-patch v5.7.0+incompatible // indirect
github.com/evanphx/json-patch/v5 v5.9.0 // indirect
github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect
github.com/fatih/color v1.13.0 // indirect
github.com/felixge/httpsnoop v1.0.3 // indirect
github.com/go-errors/errors v1.4.2 // indirect
github.com/go-gorp/gorp/v3 v3.1.0 // indirect
github.com/go-logr/logr v1.4.1 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-openapi/jsonpointer v0.19.6 // indirect
github.com/go-openapi/jsonreference v0.20.2 // indirect
github.com/go-openapi/swag v0.22.3 // indirect
github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
github.com/gobwas/glob v0.2.3 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
github.com/golang/protobuf v1.5.4 // indirect
Expand All @@ -39,25 +67,64 @@ require (
github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 // indirect
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/gorilla/mux v1.8.0 // indirect
github.com/gorilla/websocket v1.5.0 // indirect
github.com/gosuri/uitable v0.0.4 // indirect
github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect
github.com/imdario/mergo v0.3.6 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/huandu/xstrings v1.4.0 // indirect
github.com/imdario/mergo v0.3.13 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jmoiron/sqlx v1.3.5 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.16.0 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
github.com/lib/pq v1.10.9 // indirect
github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/moby/term v0.0.0-20221205130635-1aeaba878587 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.17 // indirect
github.com/mattn/go-runewidth v0.0.9 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/mitchellh/copystructure v1.2.0 // indirect
github.com/mitchellh/go-wordwrap v1.0.1 // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect
github.com/moby/locker v1.0.1 // indirect
github.com/moby/spdystream v0.2.0 // indirect
github.com/moby/term v0.5.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.0-rc6 // indirect
github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/spf13/cobra v1.7.0 // indirect
github.com/prometheus/client_golang v1.16.0 // indirect
github.com/prometheus/client_model v0.4.0 // indirect
github.com/prometheus/common v0.44.0 // indirect
github.com/prometheus/procfs v0.12.0 // indirect
github.com/rubenv/sql-migrate v1.5.2 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/shopspring/decimal v1.3.1 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/spf13/cast v1.5.0 // indirect
github.com/spf13/cobra v1.8.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
github.com/xlab/treeprint v1.2.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 // indirect
go.opentelemetry.io/otel v1.19.0 // indirect
go.opentelemetry.io/otel/metric v1.19.0 // indirect
go.opentelemetry.io/otel/trace v1.19.0 // indirect
go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect
golang.org/x/crypto v0.24.0 // indirect
golang.org/x/net v0.26.0 // indirect
Expand All @@ -69,16 +136,24 @@ require (
golang.org/x/time v0.3.0 // indirect
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect
google.golang.org/grpc v1.58.3 // indirect
google.golang.org/protobuf v1.33.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/apiextensions-apiserver v0.30.1 // indirect
k8s.io/apiserver v0.30.1 // indirect
k8s.io/cli-runtime v0.30.2 // indirect
k8s.io/client-go v0.30.2 // indirect
k8s.io/component-base v0.30.1 // indirect
k8s.io/klog/v2 v2.120.1 // indirect
k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
k8s.io/kubectl v0.30.0 // indirect
oras.land/oras-go v1.2.5 // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 // indirect
sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
sigs.k8s.io/yaml v1.3.0 // indirect
sigs.k8s.io/yaml v1.4.0 // indirect
)
Loading

0 comments on commit bb5d064

Please sign in to comment.