Fix and improve e2e test

Signed-off-by: Stefan Büringer [email protected]

test/e2e/kcp_remediations.go

@@ -19,16 +19,15 @@ package e2e
 import (
 	"context"
 	"fmt"
-	"io"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"runtime"
 	"strings"
 	"time"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	authenticationv1 "k8s.io/api/authentication/v1"
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -126,7 +125,7 @@ func KCPRemediationSpec(ctx context.Context, inputGetter func() KCPRemediationSp
 			// NOTE: this func also setups credentials/RBAC rules and everything necessary to get the authenticationToken.
 			AuthenticationToken: getAuthenticationToken(ctx, input.BootstrapClusterProxy, namespace.Name),
 			// Address to be used for accessing the management cluster from a workload cluster.
-			ServerAddr: getServerAddr(input.BootstrapClusterProxy.GetKubeconfigPath()),
+			ServerAddr: getServerAddr(ctx, input.BootstrapClusterProxy),
 		})
 
 		// The first CP machine comes up but it does not complete bootstrap
@@ -434,7 +433,7 @@ type createWorkloadClusterAndWaitInput struct {
 	SpecName             string
 	Flavor               *string
 	Namespace            string
-	AuthenticationToken  []byte
+	AuthenticationToken  string
 	ServerAddr           string
 }
 
@@ -469,7 +468,7 @@ func createWorkloadClusterAndWait(ctx context.Context, input createWorkloadClust
 		LogFolder: filepath.Join(input.ArtifactFolder, "clusters", input.Proxy.GetName()),
 		// Adds authenticationToken, server address and namespace variables to be injected in the cluster template.
 		ClusterctlVariables: map[string]string{
-			"TOKEN":     string(input.AuthenticationToken),
+			"TOKEN":     input.AuthenticationToken,
 			"SERVER":    input.ServerAddr,
 			"NAMESPACE": input.Namespace,
 		},
@@ -611,8 +610,8 @@ func waitForMachines(ctx context.Context, input waitForMachinesInput) (allMachin
 }
 
 // getServerAddr returns the address to be used for accessing the management cluster from a workload cluster.
-func getServerAddr(kubeconfigPath string) string {
-	kubeConfig, err := clientcmd.LoadFromFile(kubeconfigPath)
+func getServerAddr(ctx context.Context, clusterProxy framework.ClusterProxy) string {
+	kubeConfig, err := clientcmd.LoadFromFile(clusterProxy.GetKubeconfigPath())
 	Expect(err).ToNot(HaveOccurred(), "failed to load management cluster's kubeconfig file")
 
 	clusterName := kubeConfig.Contexts[kubeConfig.CurrentContext].Cluster
@@ -621,17 +620,39 @@ func getServerAddr(kubeconfigPath string) string {
 	serverAddr := kubeConfig.Clusters[clusterName].Server
 	Expect(serverAddr).ToNot(BeEmpty(), "failed to identify current server address in management cluster's kubeconfig file")
 
-	// On CAPD, if not running on Linux, we need to use Docker's proxy to connect back to the host
+	// With CAPD, if not running on Linux, we need to use Docker's proxy to connect back to the host
 	// to the CAPD cluster. Moby on Linux doesn't use the host.docker.internal DNS name.
 	if runtime.GOOS != "linux" {
 		serverAddr = strings.ReplaceAll(serverAddr, "127.0.0.1", "host.docker.internal")
 	}
+
+	// With CAPD, running on Linux we can't just access the bootstrap cluster via 127.0.0.1:<port> from the
+	// workload cluster. Instead we retrieve the server name from the cluster-info ConfigMap in the bootstrap
+	// cluster (e.g. "https://test-z45p9k-control-plane:6443")
+	if runtime.GOOS == "linux" {
+		clusterInfoCM := &corev1.ConfigMap{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "cluster-info",
+				Namespace: metav1.NamespacePublic,
+			},
+		}
+		Expect(clusterProxy.GetClient().Get(ctx, client.ObjectKeyFromObject(clusterInfoCM), clusterInfoCM)).To(Succeed())
+		Expect(clusterInfoCM.Data).To(HaveKey("kubeconfig"))
+
+		kubeConfigString := clusterInfoCM.Data["kubeconfig"]
+
+		kubeConfig, err := clientcmd.Load([]byte(kubeConfigString))
+		Expect(err).ToNot(HaveOccurred())
+
+		serverAddr = kubeConfig.Clusters[""].Server
+	}
+
 	return serverAddr
 }
 
 // getAuthenticationToken returns a bearer authenticationToken with minimal RBAC permissions to access the mhc-test ConfigMap that will be used
 // to control machines bootstrap during the remediation tests.
-func getAuthenticationToken(ctx context.Context, managementClusterProxy framework.ClusterProxy, namespace string) []byte {
+func getAuthenticationToken(ctx context.Context, managementClusterProxy framework.ClusterProxy, namespace string) string {
 	sa := &corev1.ServiceAccount{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "mhc-test",
@@ -677,21 +698,13 @@ func getAuthenticationToken(ctx context.Context, managementClusterProxy framewor
 	}
 	Expect(managementClusterProxy.GetClient().Create(ctx, roleBinding)).To(Succeed(), "failed to create mhc-test role binding")
 
-	cmd := exec.CommandContext(ctx, "kubectl", fmt.Sprintf("--kubeconfig=%s", managementClusterProxy.GetKubeconfigPath()), fmt.Sprintf("--namespace=%s", namespace), "create", "token", "mhc-test") //nolint:gosec
-	stdout, err := cmd.StdoutPipe()
-	Expect(err).ToNot(HaveOccurred(), "failed to get stdout for kubectl create authenticationToken")
-	stderr, err := cmd.StderrPipe()
-	Expect(err).ToNot(HaveOccurred(), "failed to get stderr for kubectl create authenticationToken")
-
-	Expect(cmd.Start()).To(Succeed(), "failed to run kubectl create authenticationToken")
-
-	output, err := io.ReadAll(stdout)
-	Expect(err).ToNot(HaveOccurred(), "failed to read stdout from kubectl create authenticationToken")
-	errout, err := io.ReadAll(stderr)
-	Expect(err).ToNot(HaveOccurred(), "failed to read stderr from kubectl create authenticationToken")
-
-	Expect(cmd.Wait()).To(Succeed(), "failed to wait kubectl create authenticationToken")
-	Expect(errout).To(BeEmpty())
+	tokenRequest := &authenticationv1.TokenRequest{
+		Spec: authenticationv1.TokenRequestSpec{
+			ExpirationSeconds: pointer.Int64(2 * 60 * 60), // 2 hours.
+		},
+	}
+	tokenRequest, err := managementClusterProxy.GetClientSet().CoreV1().ServiceAccounts(namespace).CreateToken(ctx, "mhc-test", tokenRequest, metav1.CreateOptions{})
+	Expect(err).ToNot(HaveOccurred())
 
-	return output
+	return tokenRequest.Status.Token
 }