Skip to content

Commit

Permalink
Fixed reading kubeconfig first even with --bootstrap-kubeconfig flag …
Browse files Browse the repository at this point in the history 
…passed (#606)
  • Loading branch information
Dharmjit Singh authored Jul 5, 2022
1 parent 791ea9d commit e126fab
Show file tree
Hide file tree
Showing 3 changed files with 111 additions and 38 deletions.
76 changes: 70 additions & 6 deletions agent/host_agent_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -528,12 +528,10 @@ var _ = Describe("Agent", func() {
Expect(err).NotTo(HaveOccurred())

runner = setupTestInfra(ctx, hostName, getKubeConfig().Name(), ns)
runner.CommandArgs["--bootstrap-kubeconfig"] = "/root/.byoh/config"
runner.CommandArgs["--bootstrap-kubeconfig"] = "/root/config"
byoHostContainer, err = runner.SetupByoDockerHost()
Expect(err).NotTo(HaveOccurred())

output, _, err = runner.ExecByoDockerHost(byoHostContainer)
Expect(err).NotTo(HaveOccurred())
// Clean for any CSR present
var csrList certv1.CertificateSigningRequestList
Expect(k8sClient.List(ctx, &csrList)).ShouldNot(HaveOccurred())
Expand All @@ -552,7 +550,11 @@ var _ = Describe("Agent", func() {
cleanup(runner.Context, byoHostContainer, ns, agentLogFile)
})

It("should enable the bootstrap kubeconfig flow", func() {
It("should enable the bootstrap kubeconfig flow if the ~/.byoh/config does not exist", func() {
// start agent
var err error
output, _, err = runner.ExecByoDockerHost(byoHostContainer)
Expect(err).NotTo(HaveOccurred())
defer output.Close()
f := e2e.WriteDockerLog(output, agentLogFile)
defer func() {
Expand All @@ -572,7 +574,57 @@ var _ = Describe("Agent", func() {
return false
}, time.Second*2).Should(BeTrue())
})
It("should skip bootstrap kubeconfig flow if the ~/.byoh/config exists", func() {
cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
Expect(err).ShouldNot(HaveOccurred())
// create the directory to place the kubeconfig
execCommand, err := cli.ContainerExecCreate(ctx, byoHostContainer.ID, dockertypes.ExecConfig{
AttachStdin: false,
AttachStdout: true,
AttachStderr: true,
Cmd: []string{"sh", "-c", "mkdir ${HOME}/.byoh"},
})
Expect(err).ShouldNot(HaveOccurred())
err = cli.ContainerExecStart(ctx, execCommand.ID, dockertypes.ExecStartCheck{})
Expect(err).ShouldNot(HaveOccurred())
// copy the kubeconfig
execCommand, err = cli.ContainerExecCreate(ctx, byoHostContainer.ID, dockertypes.ExecConfig{
AttachStdin: false,
AttachStdout: true,
AttachStderr: true,
Cmd: []string{"sh", "-c", "cp /root/config ${HOME}/.byoh/"},
})
Expect(err).ShouldNot(HaveOccurred())
err = cli.ContainerExecStart(ctx, execCommand.ID, dockertypes.ExecStartCheck{})
Expect(err).ShouldNot(HaveOccurred())

// start agent
output, _, err = runner.ExecByoDockerHost(byoHostContainer)
Expect(err).NotTo(HaveOccurred())

defer output.Close()
f := e2e.WriteDockerLog(output, agentLogFile)
defer func() {
deferredErr := f.Close()
if deferredErr != nil {
e2e.Showf("error closing file %s: %v", agentLogFile, deferredErr)
}
}()
Consistently(func() (done bool) {
_, err := os.Stat(agentLogFile)
if err == nil {
data, err := os.ReadFile(agentLogFile)
if err == nil && strings.Contains(string(data), "\"msg\"=\"initiated bootstrap kubeconfig flow\"") {
return false
}
}
return true
}, time.Second*2).Should(BeTrue())
})
It("should not register the BYOHost with the management cluster", func() {
// start agent
_, _, err := runner.ExecByoDockerHost(byoHostContainer)
Expect(err).NotTo(HaveOccurred())
byoHostLookupKey := types.NamespacedName{Name: hostName, Namespace: ns.Name}
createdByoHost := &infrastructurev1beta1.ByoHost{}
Consistently(func() *infrastructurev1beta1.ByoHost {
Expand All @@ -584,6 +636,9 @@ var _ = Describe("Agent", func() {
}).Should(BeNil())
})
It("should create ByoHost CSR in the management cluster", func() {
// start agent
output, _, err := runner.ExecByoDockerHost(byoHostContainer)
Expect(err).NotTo(HaveOccurred())
defer output.Close()
f := e2e.WriteDockerLog(output, agentLogFile)
defer func() {
Expand All @@ -604,6 +659,9 @@ var _ = Describe("Agent", func() {
}, 10, 1).Should(Equal(fmt.Sprintf(registration.ByohCSRNameFormat, hostName)))
})
It("should persist private key", func() {
// start agent
output, _, err := runner.ExecByoDockerHost(byoHostContainer)
Expect(err).NotTo(HaveOccurred())
defer output.Close()
fAgent := e2e.WriteDockerLog(output, agentLogFile)
defer func() {
Expand Down Expand Up @@ -646,6 +704,9 @@ var _ = Describe("Agent", func() {
Expect(os.Remove(execLogFile)).ShouldNot(HaveOccurred())
})
It("should wait for the certificate to be issued", func() {
// start agent
output, _, err := runner.ExecByoDockerHost(byoHostContainer)
Expect(err).NotTo(HaveOccurred())
defer output.Close()
f := e2e.WriteDockerLog(output, agentLogFile)
defer func() {
Expand All @@ -666,6 +727,9 @@ var _ = Describe("Agent", func() {
}, time.Second*4).Should(BeTrue())
})
It("should create kubeconfig if the csr is approved", func() {
// start agent
output, _, err := runner.ExecByoDockerHost(byoHostContainer)
Expect(err).NotTo(HaveOccurred())
defer output.Close()
fAgent := e2e.WriteDockerLog(output, agentLogFile)
defer func() {
Expand All @@ -677,8 +741,8 @@ var _ = Describe("Agent", func() {

// Approve CSR
Eventually(func() (done bool) {
byohCSR, err := clientSet.CertificatesV1().CertificateSigningRequests().Get(ctx, fmt.Sprintf(registration.ByohCSRNameFormat, hostName), metav1.GetOptions{})
if err != nil {
byohCSR, kerr := clientSet.CertificatesV1().CertificateSigningRequests().Get(ctx, fmt.Sprintf(registration.ByohCSRNameFormat, hostName), metav1.GetOptions{})
if kerr != nil {
return false
}
byohCSR.Status.Conditions = append(byohCSR.Status.Conditions, certv1.CertificateSigningRequestCondition{
Expand Down
8 changes: 5 additions & 3 deletions agent/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ package main

import (
"context"
"errors"
"flag"
"fmt"
"os"
Expand Down Expand Up @@ -145,15 +146,16 @@ func main() {
logger.Error(err, "could not determine hostname")
return
}

_, err = os.Stat(registration.GetBYOHConfigPath())
// Enable bootstrap flow if --bootstrap-kubeconfig is provided
if bootstrapKubeConfig != "" {
// and config doesn't already exists in ~/.byoh/
if bootstrapKubeConfig != "" && errors.Is(err, os.ErrNotExist) {
if err = handleBootstrapFlow(logger, hostName); err != nil {
logger.Error(err, "bootstrap flow failed")
os.Exit(1)
}
}
// Handle restart flow
// Handle restart flow or if the ~/.byoh/config already exists
config, err := registration.LoadRESTClientConfig(registration.GetBYOHConfigPath())
if err != nil {
logger.Error(err, "client config load failed")
Expand Down
65 changes: 36 additions & 29 deletions test/e2e/docker_helper.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -190,35 +190,42 @@ func (r *ByoHostRunner) copyKubeconfig(config cpConfig, listopt types.ContainerL
re := regexp.MustCompile("server:.*")
kubeconfig = re.ReplaceAll(kubeconfig, []byte("server: https://127.0.0.1:"+r.Port))

// get the $HOME env variable to set the destination path for kubeconfig
execCommand, err := r.DockerClient.ContainerExecCreate(r.Context, containers[0].ID, types.ExecConfig{
AttachStdin: false,
AttachStdout: true,
AttachStderr: true,
Cmd: []string{"sh", "-c", "echo ${HOME}"},
})
Expect(err).ShouldNot(HaveOccurred())
resp, err := r.DockerClient.ContainerExecAttach(r.Context, execCommand.ID, types.ExecStartCheck{})
Expect(err).ShouldNot(HaveOccurred())
defer resp.Close()
homeDir, err := resp.Reader.ReadString('\n')
Expect(err).ShouldNot(HaveOccurred())
homeDir = strings.TrimSuffix(homeDir, "\n")
// create the directory to place the kubeconfig
execCommand, err = r.DockerClient.ContainerExecCreate(r.Context, containers[0].ID, types.ExecConfig{
AttachStdin: false,
AttachStdout: true,
AttachStderr: true,
Cmd: []string{"sh", "-c", "mkdir ${HOME}/.byoh"},
})
Expect(err).ShouldNot(HaveOccurred())
err = r.DockerClient.ContainerExecStart(r.Context, execCommand.ID, types.ExecStartCheck{})
Expect(err).ShouldNot(HaveOccurred())

Expect(os.WriteFile(tempKubeconfigPath, kubeconfig, 0644)).NotTo(HaveOccurred()) // nolint: gosec,gomnd
config.sourcePath = tempKubeconfigPath
// SplitAfterN used to remove the unwanted special characters in the homeDir
config.destPath = strings.SplitAfterN(strings.TrimSpace(homeDir)+"/.byoh/config", "/", 2)[1] // nolint: gomnd
// If the --bootstrap-kubeconfig is not provided, the tests will use
// kubeconfig placed in ~/.byoh/config
if r.CommandArgs["--bootstrap-kubeconfig"] == "" {
// get the $HOME env variable to set the destination path for kubeconfig
execCommand, err := r.DockerClient.ContainerExecCreate(r.Context, containers[0].ID, types.ExecConfig{
AttachStdin: false,
AttachStdout: true,
AttachStderr: true,
Cmd: []string{"sh", "-c", "echo ${HOME}"},
})
Expect(err).ShouldNot(HaveOccurred())
resp, err := r.DockerClient.ContainerExecAttach(r.Context, execCommand.ID, types.ExecStartCheck{})
Expect(err).ShouldNot(HaveOccurred())
defer resp.Close()
homeDir, err := resp.Reader.ReadString('\n')
Expect(err).ShouldNot(HaveOccurred())
homeDir = strings.TrimSuffix(homeDir, "\n")
// create the directory to place the kubeconfig
execCommand, err = r.DockerClient.ContainerExecCreate(r.Context, containers[0].ID, types.ExecConfig{
AttachStdin: false,
AttachStdout: true,
AttachStderr: true,
Cmd: []string{"sh", "-c", "mkdir ${HOME}/.byoh"},
})
Expect(err).ShouldNot(HaveOccurred())
err = r.DockerClient.ContainerExecStart(r.Context, execCommand.ID, types.ExecStartCheck{})
Expect(err).ShouldNot(HaveOccurred())

Expect(os.WriteFile(tempKubeconfigPath, kubeconfig, 0644)).NotTo(HaveOccurred()) // nolint: gosec,gomnd
config.sourcePath = tempKubeconfigPath
// SplitAfterN used to remove the unwanted special characters in the homeDir
config.destPath = strings.SplitAfterN(strings.TrimSpace(homeDir)+"/.byoh/config", "/", 2)[1] // nolint: gomnd
} else {
config.sourcePath = tempKubeconfigPath
config.destPath = r.CommandArgs["--bootstrap-kubeconfig"]
}
} else {
listopt.Filters.Add("name", r.clusterConName+"-control-plane")
containers, err := r.DockerClient.ContainerList(r.Context, listopt)
Expand Down

0 comments on commit e126fab

Please sign in to comment.