From 00b281dfcd70ebfc70ae015979879c36d2272e8d Mon Sep 17 00:00:00 2001 From: Nilanjan Daw Date: Wed, 2 Mar 2022 09:41:37 +0530 Subject: [PATCH] Add support for host-agent integration tests (#400) Add support functions required for integration tests to be added for the host-agent. The integration tests will reuse functions from the E2E tests with some refactors, which are also included here. --- agent/host_agent_suite_test.go | 1 + agent/host_agent_test.go | 1 + test/e2e/byohost_reuse_test.go | 47 ++++++-- ...{e2e_docker_helper.go => docker_helper.go} | 113 ++++++++++++------ test/e2e/e2e_test.go | 52 ++++++-- test/e2e/md_scale_test.go | 26 +++- 6 files changed, 178 insertions(+), 62 deletions(-) rename test/e2e/{e2e_docker_helper.go => docker_helper.go} (65%) diff --git a/agent/host_agent_suite_test.go b/agent/host_agent_suite_test.go index cd247c8f7..ebb24b96c 100644 --- a/agent/host_agent_suite_test.go +++ b/agent/host_agent_suite_test.go @@ -53,6 +53,7 @@ var _ = BeforeSuite(func() { filepath.Join("..", "config", "crd", "bases"), filepath.Join(build.Default.GOPATH, "pkg", "mod", "sigs.k8s.io", "cluster-api@v1.0.4", "config", "crd", "bases"), }, + ErrorIfCRDPathMissing: true, } diff --git a/agent/host_agent_test.go b/agent/host_agent_test.go index f96dc497b..3730bb9c1 100644 --- a/agent/host_agent_test.go +++ b/agent/host_agent_test.go @@ -12,6 +12,7 @@ import ( "os" "os/exec" "runtime" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/test/e2e/byohost_reuse_test.go b/test/e2e/byohost_reuse_test.go index 141dd3429..8984e3483 100644 --- a/test/e2e/byohost_reuse_test.go +++ b/test/e2e/byohost_reuse_test.go @@ -14,6 +14,7 @@ import ( "github.com/docker/docker/client" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/onsi/gomega/gexec" infrastructurev1beta1 "github.com/vmware-tanzu/cluster-api-provider-bringyourownhost/apis/infrastructure/v1beta1" corev1 "k8s.io/api/core/v1" k8stypes "k8s.io/apimachinery/pkg/types" @@ -31,14 +32,16 @@ var ( var _ = Describe("When BYO Host rejoins the capacity pool", func() { var ( - ctx context.Context - specName = "byohost-reuse" - namespace *corev1.Namespace - cancelWatches context.CancelFunc - clusterResources *clusterctl.ApplyClusterTemplateAndWaitResult - byohostContainerIDs []string - agentLogFile1 = "/tmp/host-agent1.log" - agentLogFile2 = "/tmp/host-agent-reuse.log" + ctx context.Context + specName = "byohost-reuse" + namespace *corev1.Namespace + cancelWatches context.CancelFunc + clusterResources *clusterctl.ApplyClusterTemplateAndWaitResult + byohostContainerIDs []string + agentLogFile1 = "/tmp/host-agent1.log" + agentLogFile2 = "/tmp/host-agent-reuse.log" + pathToHostAgentBinary string + err error ) BeforeEach(func() { @@ -53,6 +56,9 @@ var _ = Describe("When BYO Host rejoins the capacity pool", func() { Expect(e2eConfig.Variables).To(HaveKey(KubernetesVersion)) + pathToHostAgentBinary, err = gexec.Build("github.com/vmware-tanzu/cluster-api-provider-bringyourownhost/agent") + Expect(err).NotTo(HaveOccurred()) + // set up a Namespace where to host objects for this spec and create a watcher for the namespace events. namespace, cancelWatches = setupSpecNamespace(ctx, specName, bootstrapClusterProxy, artifactFolder) clusterResources = new(clusterctl.ApplyClusterTemplateAndWaitResult) @@ -67,8 +73,26 @@ var _ = Describe("When BYO Host rejoins the capacity pool", func() { Expect(err).NotTo(HaveOccurred()) setDockerClient(client) + runner := ByoHostRunner{ + Context: ctx, + clusterConName: clusterConName, + Namespace: namespace.Name, + PathToHostAgentBinary: pathToHostAgentBinary, + DockerClient: dockerClient, + NetworkInterface: "kind", + bootstrapClusterProxy: bootstrapClusterProxy, + CommandArgs: map[string]string{ + "--kubeconfig": "/mgmt.conf", + "--namespace": namespace.Name, + "--v": "1", + }, + } + var output types.HijackedResponse - output, byohostContainerID, err := setupByoDockerHost(ctx, clusterConName, byoHostName1, namespace.Name, getDockerClient(), bootstrapClusterProxy) + runner.ByoHostName = byoHostName1 + byohost, err := runner.SetupByoDockerHost() + Expect(err).NotTo(HaveOccurred()) + output, byohostContainerID, err := runner.ExecByoDockerHost(byohost) Expect(err).NotTo(HaveOccurred()) defer output.Close() byohostContainerIDs = append(byohostContainerIDs, byohostContainerID) @@ -80,7 +104,10 @@ var _ = Describe("When BYO Host rejoins the capacity pool", func() { } }() - output, byohostContainerID, err = setupByoDockerHost(ctx, clusterConName, byoHostName2, namespace.Name, getDockerClient(), bootstrapClusterProxy) + runner.ByoHostName = byoHostName2 + byohost, err = runner.SetupByoDockerHost() + Expect(err).NotTo(HaveOccurred()) + output, byohostContainerID, err = runner.ExecByoDockerHost(byohost) Expect(err).NotTo(HaveOccurred()) defer output.Close() byohostContainerIDs = append(byohostContainerIDs, byohostContainerID) diff --git a/test/e2e/e2e_docker_helper.go b/test/e2e/docker_helper.go similarity index 65% rename from test/e2e/e2e_docker_helper.go rename to test/e2e/docker_helper.go index 7b524eb59..6e15edea5 100644 --- a/test/e2e/e2e_docker_helper.go +++ b/test/e2e/docker_helper.go @@ -20,7 +20,6 @@ import ( "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/system" . "github.com/onsi/gomega" // nolint: stylecheck - "github.com/onsi/gomega/gexec" "github.com/pkg/errors" "sigs.k8s.io/cluster-api/test/framework" ) @@ -38,6 +37,20 @@ type cpConfig struct { container string } +type ByoHostRunner struct { + Context context.Context + clusterConName string + ByoHostName string + PathToHostAgentBinary string + Namespace string + DockerClient *client.Client + NetworkInterface string + bootstrapClusterProxy framework.ClusterProxy + CommandArgs map[string]string + Port string + KubeconfigFile string +} + func resolveLocalPath(localPath string) (absPath string, err error) { if absPath, err = filepath.Abs(localPath); err != nil { return @@ -142,74 +155,98 @@ func copyToContainer(ctx context.Context, cli *client.Client, copyConfig cpConfi return cli.CopyToContainer(ctx, copyConfig.container, resolvedDstPath, content, options) } -func createDockerContainer(ctx context.Context, byoHostName string, dockerClient *client.Client) (container.ContainerCreateCreatedBody, error) { +func (r *ByoHostRunner) createDockerContainer() (container.ContainerCreateCreatedBody, error) { tmpfs := map[string]string{"/run": "", "/tmp": ""} - return dockerClient.ContainerCreate(ctx, - &container.Config{Hostname: byoHostName, + return r.DockerClient.ContainerCreate(r.Context, + &container.Config{Hostname: r.ByoHostName, Image: KindImage, }, &container.HostConfig{Privileged: true, SecurityOpt: []string{"seccomp=unconfined"}, Tmpfs: tmpfs, - NetworkMode: "kind", + NetworkMode: container.NetworkMode(r.NetworkInterface), Binds: []string{"/var", "/lib/modules:/lib/modules:ro"}, }, - &network.NetworkingConfig{EndpointsConfig: map[string]*network.EndpointSettings{"kind": {}}}, - nil, byoHostName) + &network.NetworkingConfig{EndpointsConfig: map[string]*network.EndpointSettings{r.NetworkInterface: {}}}, + nil, r.ByoHostName) } -func setupByoDockerHost(ctx context.Context, clusterConName, byoHostName, namespace string, dockerClient *client.Client, bootstrapClusterProxy framework.ClusterProxy) (types.HijackedResponse, string, error) { - byohost, err := createDockerContainer(ctx, byoHostName, dockerClient) - Expect(err).NotTo(HaveOccurred()) +func (r *ByoHostRunner) copyKubeconfig(config cpConfig, listopt types.ContainerListOptions) error { + var kubeconfig []byte + if r.NetworkInterface == "host" { + listopt.Filters.Add("name", r.ByoHostName) + containers, err := r.DockerClient.ContainerList(r.Context, listopt) + Expect(err).NotTo(HaveOccurred()) + Expect(len(containers)).To(Equal(1)) + + kubeconfig, err = os.ReadFile(r.KubeconfigFile) + Expect(err).NotTo(HaveOccurred()) + + re := regexp.MustCompile("server:.*") + kubeconfig = re.ReplaceAll(kubeconfig, []byte("server: https://127.0.0.1:"+r.Port)) + } else { + listopt.Filters.Add("name", r.clusterConName+"-control-plane") + containers, err := r.DockerClient.ContainerList(r.Context, listopt) + Expect(err).NotTo(HaveOccurred()) + Expect(len(containers)).To(Equal(1)) + + profile, err := r.DockerClient.ContainerInspect(r.Context, containers[0].ID) + Expect(err).NotTo(HaveOccurred()) + + kubeconfig, err = os.ReadFile(r.bootstrapClusterProxy.GetKubeconfigPath()) + Expect(err).NotTo(HaveOccurred()) + + re := regexp.MustCompile("server:.*") + kubeconfig = re.ReplaceAll(kubeconfig, []byte("server: https://"+profile.NetworkSettings.Networks[r.NetworkInterface].IPAddress+":6443")) + } + Expect(os.WriteFile(TempKubeconfigPath, kubeconfig, 0644)).NotTo(HaveOccurred()) // nolint: gosec,gomnd + + config.sourcePath = TempKubeconfigPath + config.destPath = r.CommandArgs["--kubeconfig"] + err := copyToContainer(r.Context, r.DockerClient, config) + return err +} + +func (r *ByoHostRunner) SetupByoDockerHost() (*container.ContainerCreateCreatedBody, error) { + var byohost container.ContainerCreateCreatedBody + var err error - Expect(dockerClient.ContainerStart(ctx, byohost.ID, types.ContainerStartOptions{})).NotTo(HaveOccurred()) + byohost, err = r.createDockerContainer() - pathToHostAgentBinary, err := gexec.Build("github.com/vmware-tanzu/cluster-api-provider-bringyourownhost/agent") Expect(err).NotTo(HaveOccurred()) + Expect(r.DockerClient.ContainerStart(r.Context, byohost.ID, types.ContainerStartOptions{})).NotTo(HaveOccurred()) config := cpConfig{ - sourcePath: pathToHostAgentBinary, + sourcePath: r.PathToHostAgentBinary, destPath: "/agent", container: byohost.ID, } - - Expect(copyToContainer(ctx, dockerClient, config)).NotTo(HaveOccurred()) + Expect(copyToContainer(r.Context, r.DockerClient, config)).NotTo(HaveOccurred()) listopt := types.ContainerListOptions{} listopt.Filters = filters.NewArgs() - listopt.Filters.Add("name", clusterConName+"-control-plane") - - containers, err := dockerClient.ContainerList(ctx, listopt) - Expect(err).NotTo(HaveOccurred()) - Expect(len(containers)).To(Equal(1)) - profile, err := dockerClient.ContainerInspect(ctx, containers[0].ID) - Expect(err).NotTo(HaveOccurred()) - - kubeconfig, err := os.ReadFile(bootstrapClusterProxy.GetKubeconfigPath()) - Expect(err).NotTo(HaveOccurred()) - - re := regexp.MustCompile("server:.*") - kubeconfig = re.ReplaceAll(kubeconfig, []byte("server: https://"+profile.NetworkSettings.Networks["kind"].IPAddress+":6443")) - - Expect(os.WriteFile(TempKubeconfigPath, kubeconfig, 0644)).NotTo(HaveOccurred()) // nolint: gosec,gomnd - - config.sourcePath = TempKubeconfigPath - config.destPath = "/mgmt.conf" - Expect(copyToContainer(ctx, dockerClient, config)).NotTo(HaveOccurred()) + err = r.copyKubeconfig(config, listopt) + return &byohost, err +} +func (r *ByoHostRunner) ExecByoDockerHost(byohost *container.ContainerCreateCreatedBody) (types.HijackedResponse, string, error) { + var cmdArgs []string + cmdArgs = append(cmdArgs, "./agent") + for flag, arg := range r.CommandArgs { + cmdArgs = append(cmdArgs, flag, arg) + } rconfig := types.ExecConfig{ AttachStdout: true, AttachStderr: true, - Cmd: []string{"./agent", "--kubeconfig", "/mgmt.conf", "--namespace", namespace}, + Cmd: cmdArgs, } - resp, err := dockerClient.ContainerExecCreate(ctx, byohost.ID, rconfig) + resp, err := r.DockerClient.ContainerExecCreate(r.Context, byohost.ID, rconfig) Expect(err).NotTo(HaveOccurred()) - output, err := dockerClient.ContainerExecAttach(ctx, resp.ID, types.ExecStartCheck{}) - + output, err := r.DockerClient.ContainerExecAttach(r.Context, resp.ID, types.ExecStartCheck{}) return output, byohost.ID, err } diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 468662be2..09bb6266f 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -7,6 +7,7 @@ package e2e import ( "context" "fmt" + "github.com/onsi/gomega/gexec" "os" "path/filepath" @@ -32,17 +33,18 @@ const ( var _ = Describe("When BYOH joins existing cluster [PR-Blocking]", func() { var ( - ctx context.Context - specName = "quick-start" - namespace *corev1.Namespace - clusterName string - cancelWatches context.CancelFunc - clusterResources *clusterctl.ApplyClusterTemplateAndWaitResult - dockerClient *client.Client - err error - byohostContainerIDs []string - agentLogFile1 = "/tmp/host-agent1.log" - agentLogFile2 = "/tmp/host-agent2.log" + ctx context.Context + specName = "quick-start" + namespace *corev1.Namespace + clusterName string + cancelWatches context.CancelFunc + clusterResources *clusterctl.ApplyClusterTemplateAndWaitResult + dockerClient *client.Client + err error + byohostContainerIDs []string + agentLogFile1 = "/tmp/host-agent1.log" + agentLogFile2 = "/tmp/host-agent2.log" + pathToHostAgentBinary string ) BeforeEach(func() { @@ -57,6 +59,9 @@ var _ = Describe("When BYOH joins existing cluster [PR-Blocking]", func() { Expect(e2eConfig.Variables).To(HaveKey(KubernetesVersion)) + pathToHostAgentBinary, err = gexec.Build("github.com/vmware-tanzu/cluster-api-provider-bringyourownhost/agent") + Expect(err).NotTo(HaveOccurred()) + // set up a Namespace where to host objects for this spec and create a watcher for the namespace events. namespace, cancelWatches = setupSpecNamespace(ctx, specName, bootstrapClusterProxy, artifactFolder) clusterResources = new(clusterctl.ApplyClusterTemplateAndWaitResult) @@ -70,8 +75,26 @@ var _ = Describe("When BYOH joins existing cluster [PR-Blocking]", func() { dockerClient, err = client.NewClientWithOpts(client.FromEnv) Expect(err).NotTo(HaveOccurred()) + runner := ByoHostRunner{ + Context: ctx, + clusterConName: clusterConName, + Namespace: namespace.Name, + PathToHostAgentBinary: pathToHostAgentBinary, + DockerClient: dockerClient, + NetworkInterface: "kind", + bootstrapClusterProxy: bootstrapClusterProxy, + CommandArgs: map[string]string{ + "--kubeconfig": "/mgmt.conf", + "--namespace": namespace.Name, + "--v": "1", + }, + } + var output types.HijackedResponse - output, byohostContainerID, err := setupByoDockerHost(ctx, clusterConName, byoHostName1, namespace.Name, dockerClient, bootstrapClusterProxy) + runner.ByoHostName = byoHostName1 + byohost, err := runner.SetupByoDockerHost() + Expect(err).NotTo(HaveOccurred()) + output, byohostContainerID, err := runner.ExecByoDockerHost(byohost) Expect(err).NotTo(HaveOccurred()) defer output.Close() byohostContainerIDs = append(byohostContainerIDs, byohostContainerID) @@ -83,7 +106,10 @@ var _ = Describe("When BYOH joins existing cluster [PR-Blocking]", func() { } }() - output, byohostContainerID, err = setupByoDockerHost(ctx, clusterConName, byoHostName2, namespace.Name, dockerClient, bootstrapClusterProxy) + runner.ByoHostName = byoHostName2 + byohost, err = runner.SetupByoDockerHost() + Expect(err).NotTo(HaveOccurred()) + output, byohostContainerID, err = runner.ExecByoDockerHost(byohost) Expect(err).NotTo(HaveOccurred()) defer output.Close() byohostContainerIDs = append(byohostContainerIDs, byohostContainerID) diff --git a/test/e2e/md_scale_test.go b/test/e2e/md_scale_test.go index 845466455..9b13f1c18 100644 --- a/test/e2e/md_scale_test.go +++ b/test/e2e/md_scale_test.go @@ -14,6 +14,7 @@ import ( "github.com/docker/docker/client" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/onsi/gomega/gexec" corev1 "k8s.io/api/core/v1" "k8s.io/utils/pointer" "sigs.k8s.io/cluster-api/test/framework" @@ -34,6 +35,8 @@ var _ = Describe("When testing MachineDeployment scale out/in", func() { byoHostName string allbyohostContainerIDs []string allAgentLogFiles []string + pathToHostAgentBinary string + err error ) BeforeEach(func() { @@ -48,6 +51,9 @@ var _ = Describe("When testing MachineDeployment scale out/in", func() { Expect(e2eConfig.Variables).To(HaveKey(KubernetesVersion)) + pathToHostAgentBinary, err = gexec.Build("github.com/vmware-tanzu/cluster-api-provider-bringyourownhost/agent") + Expect(err).NotTo(HaveOccurred()) + // set up a Namespace where to host objects for this spec and create a watcher for the namespace events. namespace, cancelWatches = setupSpecNamespace(ctx, specName, bootstrapClusterProxy, artifactFolder) clusterResources = new(clusterctl.ApplyClusterTemplateAndWaitResult) @@ -64,7 +70,25 @@ var _ = Describe("When testing MachineDeployment scale out/in", func() { for i := 0; i < byoHostCapacityPool; i++ { byoHostName = fmt.Sprintf("byohost-%s", util.RandomString(6)) - output, byohostContainerID, err := setupByoDockerHost(ctx, clusterConName, byoHostName, namespace.Name, dockerClient, bootstrapClusterProxy) + + runner := ByoHostRunner{ + Context: ctx, + clusterConName: clusterConName, + ByoHostName: byoHostName, + Namespace: namespace.Name, + PathToHostAgentBinary: pathToHostAgentBinary, + DockerClient: dockerClient, + NetworkInterface: "kind", + bootstrapClusterProxy: bootstrapClusterProxy, + CommandArgs: map[string]string{ + "--kubeconfig": "/mgmt.conf", + "--namespace": namespace.Name, + "--v": "1", + }, + } + byohost, err := runner.SetupByoDockerHost() + Expect(err).NotTo(HaveOccurred()) + output, byohostContainerID, err := runner.ExecByoDockerHost(byohost) allbyohostContainerIDs = append(allbyohostContainerIDs, byohostContainerID) Expect(err).NotTo(HaveOccurred())