Add support for host-agent integration tests (#400)

Add support functions required for integration tests to be added for the host-agent. The integration tests will reuse functions from the E2E tests with some refactors, which are also included here.

agent/host_agent_suite_test.go

@@ -53,6 +53,7 @@ var _ = BeforeSuite(func() {
 			filepath.Join("..", "config", "crd", "bases"),
 			filepath.Join(build.Default.GOPATH, "pkg", "mod", "sigs.k8s.io", "[email protected]", "config", "crd", "bases"),
 		},
+
 		ErrorIfCRDPathMissing: true,
 	}
 

agent/host_agent_test.go

@@ -12,6 +12,7 @@ import (
 	"os"
 	"os/exec"
 	"runtime"
+
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 

test/e2e/byohost_reuse_test.go

@@ -14,6 +14,7 @@ import (
 	"github.com/docker/docker/client"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+	"github.com/onsi/gomega/gexec"
 	infrastructurev1beta1 "github.com/vmware-tanzu/cluster-api-provider-bringyourownhost/apis/infrastructure/v1beta1"
 	corev1 "k8s.io/api/core/v1"
 	k8stypes "k8s.io/apimachinery/pkg/types"
@@ -31,14 +32,16 @@ var (
 var _ = Describe("When BYO Host rejoins the capacity pool", func() {
 
 	var (
-		ctx                 context.Context
-		specName            = "byohost-reuse"
-		namespace           *corev1.Namespace
-		cancelWatches       context.CancelFunc
-		clusterResources    *clusterctl.ApplyClusterTemplateAndWaitResult
-		byohostContainerIDs []string
-		agentLogFile1       = "/tmp/host-agent1.log"
-		agentLogFile2       = "/tmp/host-agent-reuse.log"
+		ctx                   context.Context
+		specName              = "byohost-reuse"
+		namespace             *corev1.Namespace
+		cancelWatches         context.CancelFunc
+		clusterResources      *clusterctl.ApplyClusterTemplateAndWaitResult
+		byohostContainerIDs   []string
+		agentLogFile1         = "/tmp/host-agent1.log"
+		agentLogFile2         = "/tmp/host-agent-reuse.log"
+		pathToHostAgentBinary string
+		err                   error
 	)
 
 	BeforeEach(func() {
@@ -53,6 +56,9 @@ var _ = Describe("When BYO Host rejoins the capacity pool", func() {
 
 		Expect(e2eConfig.Variables).To(HaveKey(KubernetesVersion))
 
+		pathToHostAgentBinary, err = gexec.Build("github.com/vmware-tanzu/cluster-api-provider-bringyourownhost/agent")
+		Expect(err).NotTo(HaveOccurred())
+
 		// set up a Namespace where to host objects for this spec and create a watcher for the namespace events.
 		namespace, cancelWatches = setupSpecNamespace(ctx, specName, bootstrapClusterProxy, artifactFolder)
 		clusterResources = new(clusterctl.ApplyClusterTemplateAndWaitResult)
@@ -67,8 +73,26 @@ var _ = Describe("When BYO Host rejoins the capacity pool", func() {
 		Expect(err).NotTo(HaveOccurred())
 		setDockerClient(client)
 
+		runner := ByoHostRunner{
+			Context:                   ctx,
+			clusterConName:        clusterConName,
+			Namespace:             namespace.Name,
+			PathToHostAgentBinary: pathToHostAgentBinary,
+			DockerClient:          dockerClient,
+			NetworkInterface:      "kind",
+			bootstrapClusterProxy: bootstrapClusterProxy,
+			CommandArgs: map[string]string{
+				"--kubeconfig": "/mgmt.conf",
+				"--namespace":  namespace.Name,
+				"--v":          "1",
+			},
+		}
+
 		var output types.HijackedResponse
-		output, byohostContainerID, err := setupByoDockerHost(ctx, clusterConName, byoHostName1, namespace.Name, getDockerClient(), bootstrapClusterProxy)
+		runner.ByoHostName = byoHostName1
+		byohost, err := runner.SetupByoDockerHost()
+		Expect(err).NotTo(HaveOccurred())
+		output, byohostContainerID, err := runner.ExecByoDockerHost(byohost)
 		Expect(err).NotTo(HaveOccurred())
 		defer output.Close()
 		byohostContainerIDs = append(byohostContainerIDs, byohostContainerID)
@@ -80,7 +104,10 @@ var _ = Describe("When BYO Host rejoins the capacity pool", func() {
 			}
 		}()
 
-		output, byohostContainerID, err = setupByoDockerHost(ctx, clusterConName, byoHostName2, namespace.Name, getDockerClient(), bootstrapClusterProxy)
+		runner.ByoHostName = byoHostName2
+		byohost, err = runner.SetupByoDockerHost()
+		Expect(err).NotTo(HaveOccurred())
+		output, byohostContainerID, err = runner.ExecByoDockerHost(byohost)
 		Expect(err).NotTo(HaveOccurred())
 		defer output.Close()
 		byohostContainerIDs = append(byohostContainerIDs, byohostContainerID)

test/e2e/e2e_docker_helper.go → test/e2e/docker_helper.go

@@ -20,7 +20,6 @@ import (
 	"github.com/docker/docker/pkg/archive"
 	"github.com/docker/docker/pkg/system"
 	. "github.com/onsi/gomega" // nolint: stylecheck
-	"github.com/onsi/gomega/gexec"
 	"github.com/pkg/errors"
 	"sigs.k8s.io/cluster-api/test/framework"
 )
@@ -38,6 +37,20 @@ type cpConfig struct {
 	container  string
 }
 
+type ByoHostRunner struct {
+	Context               context.Context
+	clusterConName        string
+	ByoHostName           string
+	PathToHostAgentBinary string
+	Namespace             string
+	DockerClient          *client.Client
+	NetworkInterface      string
+	bootstrapClusterProxy framework.ClusterProxy
+	CommandArgs           map[string]string
+	Port                  string
+	KubeconfigFile        string
+}
+
 func resolveLocalPath(localPath string) (absPath string, err error) {
 	if absPath, err = filepath.Abs(localPath); err != nil {
 		return
@@ -142,74 +155,98 @@ func copyToContainer(ctx context.Context, cli *client.Client, copyConfig cpConfi
 	return cli.CopyToContainer(ctx, copyConfig.container, resolvedDstPath, content, options)
 }
 
-func createDockerContainer(ctx context.Context, byoHostName string, dockerClient *client.Client) (container.ContainerCreateCreatedBody, error) {
+func (r *ByoHostRunner) createDockerContainer() (container.ContainerCreateCreatedBody, error) {
 	tmpfs := map[string]string{"/run": "", "/tmp": ""}
 
-	return dockerClient.ContainerCreate(ctx,
-		&container.Config{Hostname: byoHostName,
+	return r.DockerClient.ContainerCreate(r.Context,
+		&container.Config{Hostname: r.ByoHostName,
 			Image: KindImage,
 		},
 		&container.HostConfig{Privileged: true,
 			SecurityOpt: []string{"seccomp=unconfined"},
 			Tmpfs:       tmpfs,
-			NetworkMode: "kind",
+			NetworkMode: container.NetworkMode(r.NetworkInterface),
 			Binds:       []string{"/var", "/lib/modules:/lib/modules:ro"},
 		},
-		&network.NetworkingConfig{EndpointsConfig: map[string]*network.EndpointSettings{"kind": {}}},
-		nil, byoHostName)
+		&network.NetworkingConfig{EndpointsConfig: map[string]*network.EndpointSettings{r.NetworkInterface: {}}},
+		nil, r.ByoHostName)
 }
 
-func setupByoDockerHost(ctx context.Context, clusterConName, byoHostName, namespace string, dockerClient *client.Client, bootstrapClusterProxy framework.ClusterProxy) (types.HijackedResponse, string, error) {
-	byohost, err := createDockerContainer(ctx, byoHostName, dockerClient)
-	Expect(err).NotTo(HaveOccurred())
+func (r *ByoHostRunner) copyKubeconfig(config cpConfig, listopt types.ContainerListOptions) error {
+	var kubeconfig []byte
+	if r.NetworkInterface == "host" {
+		listopt.Filters.Add("name", r.ByoHostName)
+		containers, err := r.DockerClient.ContainerList(r.Context, listopt)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(len(containers)).To(Equal(1))
+
+		kubeconfig, err = os.ReadFile(r.KubeconfigFile)
+		Expect(err).NotTo(HaveOccurred())
+
+		re := regexp.MustCompile("server:.*")
+		kubeconfig = re.ReplaceAll(kubeconfig, []byte("server: https://127.0.0.1:"+r.Port))
+	} else {
+		listopt.Filters.Add("name", r.clusterConName+"-control-plane")
+		containers, err := r.DockerClient.ContainerList(r.Context, listopt)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(len(containers)).To(Equal(1))
+
+		profile, err := r.DockerClient.ContainerInspect(r.Context, containers[0].ID)
+		Expect(err).NotTo(HaveOccurred())
+
+		kubeconfig, err = os.ReadFile(r.bootstrapClusterProxy.GetKubeconfigPath())
+		Expect(err).NotTo(HaveOccurred())
+
+		re := regexp.MustCompile("server:.*")
+		kubeconfig = re.ReplaceAll(kubeconfig, []byte("server: https://"+profile.NetworkSettings.Networks[r.NetworkInterface].IPAddress+":6443"))
+	}
+	Expect(os.WriteFile(TempKubeconfigPath, kubeconfig, 0644)).NotTo(HaveOccurred()) // nolint: gosec,gomnd
+
+	config.sourcePath = TempKubeconfigPath
+	config.destPath = r.CommandArgs["--kubeconfig"]
+	err := copyToContainer(r.Context, r.DockerClient, config)
+	return err
+}
+
+func (r *ByoHostRunner) SetupByoDockerHost() (*container.ContainerCreateCreatedBody, error) {
+	var byohost container.ContainerCreateCreatedBody
+	var err error
 
-	Expect(dockerClient.ContainerStart(ctx, byohost.ID, types.ContainerStartOptions{})).NotTo(HaveOccurred())
+	byohost, err = r.createDockerContainer()
 
-	pathToHostAgentBinary, err := gexec.Build("github.com/vmware-tanzu/cluster-api-provider-bringyourownhost/agent")
 	Expect(err).NotTo(HaveOccurred())
+	Expect(r.DockerClient.ContainerStart(r.Context, byohost.ID, types.ContainerStartOptions{})).NotTo(HaveOccurred())
 
 	config := cpConfig{
-		sourcePath: pathToHostAgentBinary,
+		sourcePath: r.PathToHostAgentBinary,
 		destPath:   "/agent",
 		container:  byohost.ID,
 	}
-
-	Expect(copyToContainer(ctx, dockerClient, config)).NotTo(HaveOccurred())
+	Expect(copyToContainer(r.Context, r.DockerClient, config)).NotTo(HaveOccurred())
 
 	listopt := types.ContainerListOptions{}
 	listopt.Filters = filters.NewArgs()
-	listopt.Filters.Add("name", clusterConName+"-control-plane")
-
-	containers, err := dockerClient.ContainerList(ctx, listopt)
-	Expect(err).NotTo(HaveOccurred())
-	Expect(len(containers)).To(Equal(1))
 
-	profile, err := dockerClient.ContainerInspect(ctx, containers[0].ID)
-	Expect(err).NotTo(HaveOccurred())
-
-	kubeconfig, err := os.ReadFile(bootstrapClusterProxy.GetKubeconfigPath())
-	Expect(err).NotTo(HaveOccurred())
-
-	re := regexp.MustCompile("server:.*")
-	kubeconfig = re.ReplaceAll(kubeconfig, []byte("server: https://"+profile.NetworkSettings.Networks["kind"].IPAddress+":6443"))
-
-	Expect(os.WriteFile(TempKubeconfigPath, kubeconfig, 0644)).NotTo(HaveOccurred()) // nolint: gosec,gomnd
-
-	config.sourcePath = TempKubeconfigPath
-	config.destPath = "/mgmt.conf"
-	Expect(copyToContainer(ctx, dockerClient, config)).NotTo(HaveOccurred())
+	err = r.copyKubeconfig(config, listopt)
+	return &byohost, err
+}
 
+func (r *ByoHostRunner) ExecByoDockerHost(byohost *container.ContainerCreateCreatedBody) (types.HijackedResponse, string, error) {
+	var cmdArgs []string
+	cmdArgs = append(cmdArgs, "./agent")
+	for flag, arg := range r.CommandArgs {
+		cmdArgs = append(cmdArgs, flag, arg)
+	}
 	rconfig := types.ExecConfig{
 		AttachStdout: true,
 		AttachStderr: true,
-		Cmd:          []string{"./agent", "--kubeconfig", "/mgmt.conf", "--namespace", namespace},
+		Cmd:          cmdArgs,
 	}
 
-	resp, err := dockerClient.ContainerExecCreate(ctx, byohost.ID, rconfig)
+	resp, err := r.DockerClient.ContainerExecCreate(r.Context, byohost.ID, rconfig)
 	Expect(err).NotTo(HaveOccurred())
 
-	output, err := dockerClient.ContainerExecAttach(ctx, resp.ID, types.ExecStartCheck{})
-
+	output, err := r.DockerClient.ContainerExecAttach(r.Context, resp.ID, types.ExecStartCheck{})
 	return output, byohost.ID, err
 }
 

test/e2e/e2e_test.go

@@ -7,6 +7,7 @@ package e2e
 import (
 	"context"
 	"fmt"
+	"github.com/onsi/gomega/gexec"
 	"os"
 	"path/filepath"
 
@@ -32,17 +33,18 @@ const (
 var _ = Describe("When BYOH joins existing cluster [PR-Blocking]", func() {
 
 	var (
-		ctx                 context.Context
-		specName            = "quick-start"
-		namespace           *corev1.Namespace
-		clusterName         string
-		cancelWatches       context.CancelFunc
-		clusterResources    *clusterctl.ApplyClusterTemplateAndWaitResult
-		dockerClient        *client.Client
-		err                 error
-		byohostContainerIDs []string
-		agentLogFile1       = "/tmp/host-agent1.log"
-		agentLogFile2       = "/tmp/host-agent2.log"
+		ctx                   context.Context
+		specName              = "quick-start"
+		namespace             *corev1.Namespace
+		clusterName           string
+		cancelWatches         context.CancelFunc
+		clusterResources      *clusterctl.ApplyClusterTemplateAndWaitResult
+		dockerClient          *client.Client
+		err                   error
+		byohostContainerIDs   []string
+		agentLogFile1         = "/tmp/host-agent1.log"
+		agentLogFile2         = "/tmp/host-agent2.log"
+		pathToHostAgentBinary string
 	)
 
 	BeforeEach(func() {
@@ -57,6 +59,9 @@ var _ = Describe("When BYOH joins existing cluster [PR-Blocking]", func() {
 
 		Expect(e2eConfig.Variables).To(HaveKey(KubernetesVersion))
 
+		pathToHostAgentBinary, err = gexec.Build("github.com/vmware-tanzu/cluster-api-provider-bringyourownhost/agent")
+		Expect(err).NotTo(HaveOccurred())
+
 		// set up a Namespace where to host objects for this spec and create a watcher for the namespace events.
 		namespace, cancelWatches = setupSpecNamespace(ctx, specName, bootstrapClusterProxy, artifactFolder)
 		clusterResources = new(clusterctl.ApplyClusterTemplateAndWaitResult)
@@ -70,8 +75,26 @@ var _ = Describe("When BYOH joins existing cluster [PR-Blocking]", func() {
 		dockerClient, err = client.NewClientWithOpts(client.FromEnv)
 		Expect(err).NotTo(HaveOccurred())
 
+		runner := ByoHostRunner{
+			Context:               ctx,
+			clusterConName:        clusterConName,
+			Namespace:             namespace.Name,
+			PathToHostAgentBinary: pathToHostAgentBinary,
+			DockerClient:          dockerClient,
+			NetworkInterface:      "kind",
+			bootstrapClusterProxy: bootstrapClusterProxy,
+			CommandArgs: map[string]string{
+				"--kubeconfig": "/mgmt.conf",
+				"--namespace":  namespace.Name,
+				"--v":          "1",
+			},
+		}
+
 		var output types.HijackedResponse
-		output, byohostContainerID, err := setupByoDockerHost(ctx, clusterConName, byoHostName1, namespace.Name, dockerClient, bootstrapClusterProxy)
+		runner.ByoHostName = byoHostName1
+		byohost, err := runner.SetupByoDockerHost()
+		Expect(err).NotTo(HaveOccurred())
+		output, byohostContainerID, err := runner.ExecByoDockerHost(byohost)
 		Expect(err).NotTo(HaveOccurred())
 		defer output.Close()
 		byohostContainerIDs = append(byohostContainerIDs, byohostContainerID)
@@ -83,7 +106,10 @@ var _ = Describe("When BYOH joins existing cluster [PR-Blocking]", func() {
 			}
 		}()
 
-		output, byohostContainerID, err = setupByoDockerHost(ctx, clusterConName, byoHostName2, namespace.Name, dockerClient, bootstrapClusterProxy)
+		runner.ByoHostName = byoHostName2
+		byohost, err = runner.SetupByoDockerHost()
+		Expect(err).NotTo(HaveOccurred())
+		output, byohostContainerID, err = runner.ExecByoDockerHost(byohost)
 		Expect(err).NotTo(HaveOccurred())
 		defer output.Close()
 		byohostContainerIDs = append(byohostContainerIDs, byohostContainerID)