From 8f6c71a3a0588bcc47437e0c668bf6e4168c1310 Mon Sep 17 00:00:00 2001 From: Stefan Bueringer Date: Tue, 7 Feb 2023 17:14:24 +0100 Subject: [PATCH] Fix and improve e2e test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stefan Büringer buringerst@vmware.com --- test/e2e/kcp_remediations.go | 63 ++++++++++++++++++++++-------------- 1 file changed, 38 insertions(+), 25 deletions(-) diff --git a/test/e2e/kcp_remediations.go b/test/e2e/kcp_remediations.go index c0e69cd0a0bc..30fca00e12b5 100644 --- a/test/e2e/kcp_remediations.go +++ b/test/e2e/kcp_remediations.go @@ -19,9 +19,7 @@ package e2e import ( "context" "fmt" - "io" "os" - "os/exec" "path/filepath" "runtime" "strings" @@ -29,6 +27,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + authenticationv1 "k8s.io/api/authentication/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -126,7 +125,7 @@ func KCPRemediationSpec(ctx context.Context, inputGetter func() KCPRemediationSp // NOTE: this func also setups credentials/RBAC rules and everything necessary to get the authenticationToken. AuthenticationToken: getAuthenticationToken(ctx, input.BootstrapClusterProxy, namespace.Name), // Address to be used for accessing the management cluster from a workload cluster. - ServerAddr: getServerAddr(input.BootstrapClusterProxy.GetKubeconfigPath()), + ServerAddr: getServerAddr(ctx, input.BootstrapClusterProxy), }) // The first CP machine comes up but it does not complete bootstrap @@ -434,7 +433,7 @@ type createWorkloadClusterAndWaitInput struct { SpecName string Flavor *string Namespace string - AuthenticationToken []byte + AuthenticationToken string ServerAddr string } @@ -469,7 +468,7 @@ func createWorkloadClusterAndWait(ctx context.Context, input createWorkloadClust LogFolder: filepath.Join(input.ArtifactFolder, "clusters", input.Proxy.GetName()), // Adds authenticationToken, server address and namespace variables to be injected in the cluster template. ClusterctlVariables: map[string]string{ - "TOKEN": string(input.AuthenticationToken), + "TOKEN": input.AuthenticationToken, "SERVER": input.ServerAddr, "NAMESPACE": input.Namespace, }, @@ -611,8 +610,8 @@ func waitForMachines(ctx context.Context, input waitForMachinesInput) (allMachin } // getServerAddr returns the address to be used for accessing the management cluster from a workload cluster. -func getServerAddr(kubeconfigPath string) string { - kubeConfig, err := clientcmd.LoadFromFile(kubeconfigPath) +func getServerAddr(ctx context.Context, clusterProxy framework.ClusterProxy) string { + kubeConfig, err := clientcmd.LoadFromFile(clusterProxy.GetKubeconfigPath()) Expect(err).ToNot(HaveOccurred(), "failed to load management cluster's kubeconfig file") clusterName := kubeConfig.Contexts[kubeConfig.CurrentContext].Cluster @@ -621,17 +620,39 @@ func getServerAddr(kubeconfigPath string) string { serverAddr := kubeConfig.Clusters[clusterName].Server Expect(serverAddr).ToNot(BeEmpty(), "failed to identify current server address in management cluster's kubeconfig file") - // On CAPD, if not running on Linux, we need to use Docker's proxy to connect back to the host + // With CAPD, if not running on Linux, we need to use Docker's proxy to connect back to the host // to the CAPD cluster. Moby on Linux doesn't use the host.docker.internal DNS name. if runtime.GOOS != "linux" { serverAddr = strings.ReplaceAll(serverAddr, "127.0.0.1", "host.docker.internal") } + + // With CAPD, running on Linux we can't just access the bootstrap cluster via 127.0.0.1: from the + // workload cluster. Instead we retrieve the server name from the cluster-info ConfigMap in the bootstrap + // cluster (e.g. "https://test-z45p9k-control-plane:6443") + if runtime.GOOS == "linux" { + clusterInfoCM := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster-info", + Namespace: metav1.NamespacePublic, + }, + } + Expect(clusterProxy.GetClient().Get(ctx, client.ObjectKeyFromObject(clusterInfoCM), clusterInfoCM)).To(Succeed()) + Expect(clusterInfoCM.Data).To(HaveKey("kubeconfig")) + + kubeConfigString := clusterInfoCM.Data["kubeconfig"] + + kubeConfig, err := clientcmd.Load([]byte(kubeConfigString)) + Expect(err).ToNot(HaveOccurred()) + + serverAddr = kubeConfig.Clusters[""].Server + } + return serverAddr } // getAuthenticationToken returns a bearer authenticationToken with minimal RBAC permissions to access the mhc-test ConfigMap that will be used // to control machines bootstrap during the remediation tests. -func getAuthenticationToken(ctx context.Context, managementClusterProxy framework.ClusterProxy, namespace string) []byte { +func getAuthenticationToken(ctx context.Context, managementClusterProxy framework.ClusterProxy, namespace string) string { sa := &corev1.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ Name: "mhc-test", @@ -677,21 +698,13 @@ func getAuthenticationToken(ctx context.Context, managementClusterProxy framewor } Expect(managementClusterProxy.GetClient().Create(ctx, roleBinding)).To(Succeed(), "failed to create mhc-test role binding") - cmd := exec.CommandContext(ctx, "kubectl", fmt.Sprintf("--kubeconfig=%s", managementClusterProxy.GetKubeconfigPath()), fmt.Sprintf("--namespace=%s", namespace), "create", "token", "mhc-test") //nolint:gosec - stdout, err := cmd.StdoutPipe() - Expect(err).ToNot(HaveOccurred(), "failed to get stdout for kubectl create authenticationToken") - stderr, err := cmd.StderrPipe() - Expect(err).ToNot(HaveOccurred(), "failed to get stderr for kubectl create authenticationToken") - - Expect(cmd.Start()).To(Succeed(), "failed to run kubectl create authenticationToken") - - output, err := io.ReadAll(stdout) - Expect(err).ToNot(HaveOccurred(), "failed to read stdout from kubectl create authenticationToken") - errout, err := io.ReadAll(stderr) - Expect(err).ToNot(HaveOccurred(), "failed to read stderr from kubectl create authenticationToken") - - Expect(cmd.Wait()).To(Succeed(), "failed to wait kubectl create authenticationToken") - Expect(errout).To(BeEmpty()) + tokenRequest := &authenticationv1.TokenRequest{ + Spec: authenticationv1.TokenRequestSpec{ + ExpirationSeconds: pointer.Int64(2 * 60 * 60), // 2 hours. + }, + } + tokenRequest, err := managementClusterProxy.GetClientSet().CoreV1().ServiceAccounts(namespace).CreateToken(ctx, "mhc-test", tokenRequest, metav1.CreateOptions{}) + Expect(err).ToNot(HaveOccurred()) - return output + return tokenRequest.Status.Token }