From eebff6f8ea4d565b63fde21fb8001454fa0219f1 Mon Sep 17 00:00:00 2001 From: Dharmjit Singh Date: Tue, 21 Jun 2022 10:18:29 +0000 Subject: [PATCH] Fixed reading kubeconfig first even with --bootstrap-kubeconfig flag passed --- agent/host_agent_test.go | 76 +++++++++++++++++++++++++++++++++++---- agent/main.go | 8 +++-- test/e2e/docker_helper.go | 65 ++++++++++++++++++--------------- 3 files changed, 111 insertions(+), 38 deletions(-) diff --git a/agent/host_agent_test.go b/agent/host_agent_test.go index 37d35ec53..a5a6a6855 100644 --- a/agent/host_agent_test.go +++ b/agent/host_agent_test.go @@ -517,12 +517,10 @@ var _ = Describe("Agent", func() { Expect(err).NotTo(HaveOccurred()) runner = setupTestInfra(ctx, hostName, getKubeConfig().Name(), ns) - runner.CommandArgs["--bootstrap-kubeconfig"] = "/root/.byoh/config" + runner.CommandArgs["--bootstrap-kubeconfig"] = "/root/config" byoHostContainer, err = runner.SetupByoDockerHost() Expect(err).NotTo(HaveOccurred()) - output, _, err = runner.ExecByoDockerHost(byoHostContainer) - Expect(err).NotTo(HaveOccurred()) // Clean for any CSR present var csrList certv1.CertificateSigningRequestList Expect(k8sClient.List(ctx, &csrList)).ShouldNot(HaveOccurred()) @@ -541,7 +539,11 @@ var _ = Describe("Agent", func() { cleanup(runner.Context, byoHostContainer, ns, agentLogFile) }) - It("should enable the bootstrap kubeconfig flow", func() { + It("should enable the bootstrap kubeconfig flow if the ~/.byoh/config does not exist", func() { + // start agent + var err error + output, _, err = runner.ExecByoDockerHost(byoHostContainer) + Expect(err).NotTo(HaveOccurred()) defer output.Close() f := e2e.WriteDockerLog(output, agentLogFile) defer func() { @@ -561,7 +563,57 @@ var _ = Describe("Agent", func() { return false }, time.Second*2).Should(BeTrue()) }) + It("should skip bootstrap kubeconfig flow if the ~/.byoh/config exists", func() { + cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) + Expect(err).ShouldNot(HaveOccurred()) + // create the directory to place the kubeconfig + execCommand, err := cli.ContainerExecCreate(ctx, byoHostContainer.ID, dockertypes.ExecConfig{ + AttachStdin: false, + AttachStdout: true, + AttachStderr: true, + Cmd: []string{"sh", "-c", "mkdir ${HOME}/.byoh"}, + }) + Expect(err).ShouldNot(HaveOccurred()) + err = cli.ContainerExecStart(ctx, execCommand.ID, dockertypes.ExecStartCheck{}) + Expect(err).ShouldNot(HaveOccurred()) + // copy the kubeconfig + execCommand, err = cli.ContainerExecCreate(ctx, byoHostContainer.ID, dockertypes.ExecConfig{ + AttachStdin: false, + AttachStdout: true, + AttachStderr: true, + Cmd: []string{"sh", "-c", "cp /root/config ${HOME}/.byoh/"}, + }) + Expect(err).ShouldNot(HaveOccurred()) + err = cli.ContainerExecStart(ctx, execCommand.ID, dockertypes.ExecStartCheck{}) + Expect(err).ShouldNot(HaveOccurred()) + + // start agent + output, _, err = runner.ExecByoDockerHost(byoHostContainer) + Expect(err).NotTo(HaveOccurred()) + + defer output.Close() + f := e2e.WriteDockerLog(output, agentLogFile) + defer func() { + deferredErr := f.Close() + if deferredErr != nil { + e2e.Showf("error closing file %s: %v", agentLogFile, deferredErr) + } + }() + Consistently(func() (done bool) { + _, err := os.Stat(agentLogFile) + if err == nil { + data, err := os.ReadFile(agentLogFile) + if err == nil && strings.Contains(string(data), "\"msg\"=\"initiated bootstrap kubeconfig flow\"") { + return false + } + } + return true + }, time.Second*2).Should(BeTrue()) + }) It("should not register the BYOHost with the management cluster", func() { + // start agent + _, _, err := runner.ExecByoDockerHost(byoHostContainer) + Expect(err).NotTo(HaveOccurred()) byoHostLookupKey := types.NamespacedName{Name: hostName, Namespace: ns.Name} createdByoHost := &infrastructurev1beta1.ByoHost{} Consistently(func() *infrastructurev1beta1.ByoHost { @@ -573,6 +625,9 @@ var _ = Describe("Agent", func() { }).Should(BeNil()) }) It("should create ByoHost CSR in the management cluster", func() { + // start agent + output, _, err := runner.ExecByoDockerHost(byoHostContainer) + Expect(err).NotTo(HaveOccurred()) defer output.Close() f := e2e.WriteDockerLog(output, agentLogFile) defer func() { @@ -593,6 +648,9 @@ var _ = Describe("Agent", func() { }, 10, 1).Should(Equal(fmt.Sprintf(registration.ByohCSRNameFormat, hostName))) }) It("should persist private key", func() { + // start agent + output, _, err := runner.ExecByoDockerHost(byoHostContainer) + Expect(err).NotTo(HaveOccurred()) defer output.Close() fAgent := e2e.WriteDockerLog(output, agentLogFile) defer func() { @@ -635,6 +693,9 @@ var _ = Describe("Agent", func() { Expect(os.Remove(execLogFile)).ShouldNot(HaveOccurred()) }) It("should wait for the certificate to be issued", func() { + // start agent + output, _, err := runner.ExecByoDockerHost(byoHostContainer) + Expect(err).NotTo(HaveOccurred()) defer output.Close() f := e2e.WriteDockerLog(output, agentLogFile) defer func() { @@ -655,6 +716,9 @@ var _ = Describe("Agent", func() { }, time.Second*4).Should(BeTrue()) }) It("should create kubeconfig if the csr is approved", func() { + // start agent + output, _, err := runner.ExecByoDockerHost(byoHostContainer) + Expect(err).NotTo(HaveOccurred()) defer output.Close() fAgent := e2e.WriteDockerLog(output, agentLogFile) defer func() { @@ -666,8 +730,8 @@ var _ = Describe("Agent", func() { // Approve CSR Eventually(func() (done bool) { - byohCSR, err := clientSet.CertificatesV1().CertificateSigningRequests().Get(ctx, fmt.Sprintf(registration.ByohCSRNameFormat, hostName), metav1.GetOptions{}) - if err != nil { + byohCSR, kerr := clientSet.CertificatesV1().CertificateSigningRequests().Get(ctx, fmt.Sprintf(registration.ByohCSRNameFormat, hostName), metav1.GetOptions{}) + if kerr != nil { return false } byohCSR.Status.Conditions = append(byohCSR.Status.Conditions, certv1.CertificateSigningRequestCondition{ diff --git a/agent/main.go b/agent/main.go index a6a6a5508..69db8f36e 100644 --- a/agent/main.go +++ b/agent/main.go @@ -5,6 +5,7 @@ package main import ( "context" + "errors" "flag" "fmt" "os" @@ -149,15 +150,16 @@ func main() { logger.Error(err, "could not determine hostname") return } - + _, err = os.Stat(registration.GetBYOHConfigPath()) // Enable bootstrap flow if --bootstrap-kubeconfig is provided - if bootstrapKubeConfig != "" { + // and config doesn't already exists in ~/.byoh/ + if bootstrapKubeConfig != "" && errors.Is(err, os.ErrNotExist) { if err = handleBootstrapFlow(logger, hostName); err != nil { logger.Error(err, "bootstrap flow failed") os.Exit(1) } } - // Handle restart flow + // Handle restart flow or if the ~/.byoh/config already exists config, err := registration.LoadRESTClientConfig(registration.GetBYOHConfigPath()) if err != nil { logger.Error(err, "client config load failed") diff --git a/test/e2e/docker_helper.go b/test/e2e/docker_helper.go index f791fc022..055a4870d 100644 --- a/test/e2e/docker_helper.go +++ b/test/e2e/docker_helper.go @@ -191,35 +191,42 @@ func (r *ByoHostRunner) copyKubeconfig(config cpConfig, listopt types.ContainerL re := regexp.MustCompile("server:.*") kubeconfig = re.ReplaceAll(kubeconfig, []byte("server: https://127.0.0.1:"+r.Port)) - // get the $HOME env variable to set the destination path for kubeconfig - execCommand, err := r.DockerClient.ContainerExecCreate(r.Context, containers[0].ID, types.ExecConfig{ - AttachStdin: false, - AttachStdout: true, - AttachStderr: true, - Cmd: []string{"sh", "-c", "echo ${HOME}"}, - }) - Expect(err).ShouldNot(HaveOccurred()) - resp, err := r.DockerClient.ContainerExecAttach(r.Context, execCommand.ID, types.ExecStartCheck{}) - Expect(err).ShouldNot(HaveOccurred()) - defer resp.Close() - homeDir, err := resp.Reader.ReadString('\n') - Expect(err).ShouldNot(HaveOccurred()) - homeDir = strings.TrimSuffix(homeDir, "\n") - // create the directory to place the kubeconfig - execCommand, err = r.DockerClient.ContainerExecCreate(r.Context, containers[0].ID, types.ExecConfig{ - AttachStdin: false, - AttachStdout: true, - AttachStderr: true, - Cmd: []string{"sh", "-c", "mkdir ${HOME}/.byoh"}, - }) - Expect(err).ShouldNot(HaveOccurred()) - err = r.DockerClient.ContainerExecStart(r.Context, execCommand.ID, types.ExecStartCheck{}) - Expect(err).ShouldNot(HaveOccurred()) - - Expect(os.WriteFile(tempKubeconfigPath, kubeconfig, 0644)).NotTo(HaveOccurred()) // nolint: gosec,gomnd - config.sourcePath = tempKubeconfigPath - // SplitAfterN used to remove the unwanted special characters in the homeDir - config.destPath = strings.SplitAfterN(strings.TrimSpace(homeDir)+"/.byoh/config", "/", 2)[1] // nolint: gomnd + // If the --bootstrap-kubeconfig is not provided, the tests will use + // kubeconfig placed in ~/.byoh/config + if r.CommandArgs["--bootstrap-kubeconfig"] == "" { + // get the $HOME env variable to set the destination path for kubeconfig + execCommand, err := r.DockerClient.ContainerExecCreate(r.Context, containers[0].ID, types.ExecConfig{ + AttachStdin: false, + AttachStdout: true, + AttachStderr: true, + Cmd: []string{"sh", "-c", "echo ${HOME}"}, + }) + Expect(err).ShouldNot(HaveOccurred()) + resp, err := r.DockerClient.ContainerExecAttach(r.Context, execCommand.ID, types.ExecStartCheck{}) + Expect(err).ShouldNot(HaveOccurred()) + defer resp.Close() + homeDir, err := resp.Reader.ReadString('\n') + Expect(err).ShouldNot(HaveOccurred()) + homeDir = strings.TrimSuffix(homeDir, "\n") + // create the directory to place the kubeconfig + execCommand, err = r.DockerClient.ContainerExecCreate(r.Context, containers[0].ID, types.ExecConfig{ + AttachStdin: false, + AttachStdout: true, + AttachStderr: true, + Cmd: []string{"sh", "-c", "mkdir ${HOME}/.byoh"}, + }) + Expect(err).ShouldNot(HaveOccurred()) + err = r.DockerClient.ContainerExecStart(r.Context, execCommand.ID, types.ExecStartCheck{}) + Expect(err).ShouldNot(HaveOccurred()) + + Expect(os.WriteFile(tempKubeconfigPath, kubeconfig, 0644)).NotTo(HaveOccurred()) // nolint: gosec,gomnd + config.sourcePath = tempKubeconfigPath + // SplitAfterN used to remove the unwanted special characters in the homeDir + config.destPath = strings.SplitAfterN(strings.TrimSpace(homeDir)+"/.byoh/config", "/", 2)[1] // nolint: gomnd + } else { + config.sourcePath = tempKubeconfigPath + config.destPath = r.CommandArgs["--bootstrap-kubeconfig"] + } } else { listopt.Filters.Add("name", r.clusterConName+"-control-plane") containers, err := r.DockerClient.ContainerList(r.Context, listopt)