diff --git a/test/e2e/cloud-provider-azure.go b/test/e2e/cloud-provider-azure.go index b6bfc19f851..cf89d2384a6 100644 --- a/test/e2e/cloud-provider-azure.go +++ b/test/e2e/cloud-provider-azure.go @@ -23,122 +23,47 @@ import ( "context" "fmt" - "github.com/Azure/go-autorest/autorest/to" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - helmAction "helm.sh/helm/v3/pkg/action" - helmLoader "helm.sh/helm/v3/pkg/chart/loader" - helmCli "helm.sh/helm/v3/pkg/cli" - helmVals "helm.sh/helm/v3/pkg/cli/values" - helmGetter "helm.sh/helm/v3/pkg/getter" - corev1 "k8s.io/api/core/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/cluster-api/test/framework/clusterctl" - crclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + cloudProviderAzureHelmRepoURL = "https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo" + cloudProviderAzureChartName = "cloud-provider-azure" + cloudProvierAzureHelmReleaseName = "cloud-provider-azure-oot" ) // InstallCloudProviderAzureHelmChart installs the official cloud-provider-azure helm chart +// Fulfills the clusterctl.Waiter type so that it can be used as ApplyClusterTemplateAndWaitInput data +// in the flow of a clusterctl.ApplyClusterTemplateAndWait E2E test scenario func InstallCloudProviderAzureHelmChart(ctx context.Context, input clusterctl.ApplyClusterTemplateAndWaitInput, result *clusterctl.ApplyClusterTemplateAndWaitResult) { - By("Waiting for workload cluster kubeconfig secret") - Eventually(func() error { - client := input.ClusterProxy.GetClient() - secret := &corev1.Secret{} - key := crclient.ObjectKey{ - Name: fmt.Sprintf("%s-kubeconfig", input.ConfigCluster.ClusterName), - Namespace: input.ConfigCluster.Namespace, - } - return client.Get(ctx, key, secret) - }, input.WaitForControlPlaneIntervals...).Should(Succeed()) - clusterProxy := input.ClusterProxy.GetWorkloadCluster(ctx, input.ConfigCluster.Namespace, input.ConfigCluster.ClusterName) - By("Waiting for nodes to come online indicating that the cluster is ready to accept work") - Eventually(func() error { - clientSet := clusterProxy.GetClientSet() - var runningNodes int - list, err := clientSet.CoreV1().Nodes().List(ctx, v1.ListOptions{}) - if err != nil { - return err - } - for _, n := range list.Items { - if n.Status.Phase == corev1.NodeRunning { - runningNodes++ - } - } - if runningNodes > 0 { - return nil - } - return err - }, input.WaitForControlPlaneIntervals...).Should(Succeed()) + By(fmt.Sprintf("Ensuring the kubeconfig secret for cluster %s/%s exists before installing cloud-provider-azure components", input.ConfigCluster.Namespace, input.ConfigCluster.ClusterName)) + WaitForWorkloadClusterKubeconfigSecret(ctx, input) By("Installing the correct version of cloud-provider-azure components via helm") - kubeConfigPath := clusterProxy.GetKubeconfigPath() - clusterName := input.ClusterProxy.GetName() - settings := helmCli.New() - settings.KubeConfig = kubeConfigPath - actionConfig := new(helmAction.Configuration) - err := actionConfig.Init(settings.RESTClientGetter(), "default", "secret", Logf) - Expect(err).To(BeNil()) - i := helmAction.NewInstall(actionConfig) - i.RepoURL = "https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo" - i.ReleaseName = "cloud-provider-azure-oot" - Eventually(func() error { - cp, err := i.ChartPathOptions.LocateChart("cloud-provider-azure", helmCli.New()) - if err != nil { - return err - } - p := helmGetter.All(settings) - valueOpts := &helmVals.Options{} - valueOpts.Values = []string{fmt.Sprintf("infra.clusterName=%s", clusterName)} - vals, err := valueOpts.MergeValues(p) - if err != nil { - return err - } - chartRequested, err := helmLoader.Load(cp) - if err != nil { - return err - } - release, err := i.RunWithContext(ctx, chartRequested, vals) - if err != nil { - return err - } - Logf(release.Info.Description) - return nil - }, input.WaitForControlPlaneIntervals...).Should(Succeed()) + values := []string{fmt.Sprintf("infra.clusterName=%s", input.ConfigCluster.ClusterName)} + InstallHelmChart(ctx, input, cloudProviderAzureHelmRepoURL, cloudProviderAzureChartName, cloudProvierAzureHelmReleaseName, values) By("Waiting for a Running cloud-controller-manager pod") - Eventually(func() bool { - clusterProxy := input.ClusterProxy.GetWorkloadCluster(ctx, input.ConfigCluster.Namespace, input.ConfigCluster.ClusterName) - clientSet := clusterProxy.GetClientSet() - var runningPods int - list, err := clientSet.CoreV1().Pods("kube-system").List(ctx, v1.ListOptions{ - LabelSelector: "component=cloud-controller-manager", - }) - if err != nil { - return false - } - for _, p := range list.Items { - if p.Status.Phase == corev1.PodRunning { - runningPods++ - } - } - return runningPods > 0 - }, input.WaitForControlPlaneIntervals...).Should(BeTrue()) - By("Waiting for Running cloud-node-manager pods") - Eventually(func() bool { - clusterProxy := input.ClusterProxy.GetWorkloadCluster(ctx, input.ConfigCluster.Namespace, input.ConfigCluster.ClusterName) - clientSet := clusterProxy.GetClientSet() - var runningPods int64 - list, err := clientSet.CoreV1().Pods("kube-system").List(ctx, v1.ListOptions{ - LabelSelector: "k8s-app=cloud-node-manager", - }) - if err != nil { - return false - } - for _, p := range list.Items { - if p.Status.Phase == corev1.PodRunning { - runningPods++ - } - } - return runningPods >= to.Int64(input.ConfigCluster.ControlPlaneMachineCount) - }, input.WaitForControlPlaneIntervals...).Should(BeTrue()) + clusterProxy := input.ClusterProxy.GetWorkloadCluster(ctx, input.ConfigCluster.Namespace, input.ConfigCluster.ClusterName) + workloadClusterClient := clusterProxy.GetClient() + cloudControllerManagerPodLabel, err := labels.Parse("component=cloud-controller-manager") + Expect(err).ToNot(HaveOccurred()) + framework.WaitForPodListCondition(ctx, framework.WaitForPodListConditionInput{ + Lister: workloadClusterClient, + ListOptions: &client.ListOptions{ + LabelSelector: cloudControllerManagerPodLabel, + Namespace: "kube-system", + }, + Condition: podListHasNumPods(1), + }, input.WaitForControlPlaneIntervals...) + Expect(err).ToNot(HaveOccurred()) + By(fmt.Sprintf("Waiting for Ready cloud-node-manager daemonset pods")) + for _, ds := range []string{"cloud-node-manager", "cloud-node-manager-windows"} { + WaitForDaemonset(ctx, input, workloadClusterClient, ds, "kube-system") + } By("Done installing cloud-provider-azure components, ensuring control plane is initialized") result.ControlPlane = framework.DiscoveryAndWaitForControlPlaneInitialized(ctx, framework.DiscoveryAndWaitForControlPlaneInitializedInput{ Lister: input.ClusterProxy.GetClient(), diff --git a/test/e2e/helpers.go b/test/e2e/helpers.go index 68a562e56a8..dc199729bd2 100644 --- a/test/e2e/helpers.go +++ b/test/e2e/helpers.go @@ -66,6 +66,12 @@ import ( "sigs.k8s.io/cluster-api/test/framework/kubernetesversions" "sigs.k8s.io/cluster-api/util" "sigs.k8s.io/controller-runtime/pkg/client" + + helmAction "helm.sh/helm/v3/pkg/action" + helmLoader "helm.sh/helm/v3/pkg/chart/loader" + helmCli "helm.sh/helm/v3/pkg/cli" + helmVals "helm.sh/helm/v3/pkg/cli/values" + helmGetter "helm.sh/helm/v3/pkg/getter" ) const ( @@ -823,3 +829,90 @@ func getPodLogs(ctx context.Context, clientset *kubernetes.Clientset, pod corev1 } return b.String() } + +// InstallHelmChart takes a helm repo URL, a chart name, and release name, and installs a helm release onto the E2E workload cluster +func InstallHelmChart(ctx context.Context, input clusterctl.ApplyClusterTemplateAndWaitInput, repoURL, chartName, releaseName string, values []string) { + clusterProxy := input.ClusterProxy.GetWorkloadCluster(ctx, input.ConfigCluster.Namespace, input.ConfigCluster.ClusterName) + kubeConfigPath := clusterProxy.GetKubeconfigPath() + settings := helmCli.New() + settings.KubeConfig = kubeConfigPath + actionConfig := new(helmAction.Configuration) + err := actionConfig.Init(settings.RESTClientGetter(), "default", "secret", Logf) + Expect(err).To(BeNil()) + i := helmAction.NewInstall(actionConfig) + i.RepoURL = repoURL + i.ReleaseName = releaseName + Eventually(func() error { + cp, err := i.ChartPathOptions.LocateChart(chartName, helmCli.New()) + if err != nil { + return err + } + p := helmGetter.All(settings) + valueOpts := &helmVals.Options{} + valueOpts.Values = values + vals, err := valueOpts.MergeValues(p) + if err != nil { + return err + } + chartRequested, err := helmLoader.Load(cp) + if err != nil { + return err + } + release, err := i.RunWithContext(ctx, chartRequested, vals) + if err != nil { + return err + } + Logf(release.Info.Description) + return nil + }, input.WaitForControlPlaneIntervals...).Should(Succeed()) +} + +// WaitForWorkloadClusterKubeconfigSecret retries during E2E until the workload cluster kubeconfig secret exists +func WaitForWorkloadClusterKubeconfigSecret(ctx context.Context, input clusterctl.ApplyClusterTemplateAndWaitInput) { + // Ensure the workload cluster kubeconfig secret exists before getting the workload cluster clusterProxy object + Eventually(func() error { + cl := input.ClusterProxy.GetClient() + secret := &corev1.Secret{} + key := client.ObjectKey{ + Name: fmt.Sprintf("%s-kubeconfig", input.ConfigCluster.ClusterName), + Namespace: input.ConfigCluster.Namespace, + } + err := cl.Get(ctx, key, secret) + if err != nil { + return err + } + return nil + }, input.WaitForControlPlaneIntervals...).Should(Succeed()) +} + +// WaitForDaemonset retries during E2E until a daemonset's pods are all Running +func WaitForDaemonset(ctx context.Context, input clusterctl.ApplyClusterTemplateAndWaitInput, cl client.Client, name, namespace string) { + // Ensure the workload cluster kubeconfig secret exists before getting the workload cluster clusterProxy object + Eventually(func() bool { + ds := &appsv1.DaemonSet{} + if err := cl.Get(ctx, client.ObjectKey{Name: name, Namespace: namespace}, ds); err != nil { + return false + } + if ds.Status.DesiredNumberScheduled == ds.Status.NumberReady { + return true + } + return false + }, input.WaitForControlPlaneIntervals...).Should(Equal(true)) +} + +// podListHasNumPods fulfills the cluster-api PodListCondition type spec +// given a list of pods, we validate for an exact number of those pods in a Running state +func podListHasNumPods(numPods int) func(pl *corev1.PodList) error { + return func(pl *corev1.PodList) error { + var runningPods int + for _, p := range pl.Items { + if p.Status.Phase == corev1.PodRunning { + runningPods++ + } + } + if runningPods != numPods { + return errors.Errorf("expected %d Running pods, got %d", numPods, runningPods) + } + return nil + } +}