Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix: Read kubeconfig first even with --bootstrap-kubeconfig flag #606

Merged
merged 1 commit into from
Jul 5, 2022
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 70 additions & 6 deletions agent/host_agent_test.go
Original file line number Diff line number Diff line change
@@ -517,12 +517,10 @@ var _ = Describe("Agent", func() {
Expect(err).NotTo(HaveOccurred())

runner = setupTestInfra(ctx, hostName, getKubeConfig().Name(), ns)
runner.CommandArgs["--bootstrap-kubeconfig"] = "/root/.byoh/config"
runner.CommandArgs["--bootstrap-kubeconfig"] = "/root/config"
byoHostContainer, err = runner.SetupByoDockerHost()
Expect(err).NotTo(HaveOccurred())

output, _, err = runner.ExecByoDockerHost(byoHostContainer)
Expect(err).NotTo(HaveOccurred())
// Clean for any CSR present
var csrList certv1.CertificateSigningRequestList
Expect(k8sClient.List(ctx, &csrList)).ShouldNot(HaveOccurred())
@@ -541,7 +539,11 @@ var _ = Describe("Agent", func() {
cleanup(runner.Context, byoHostContainer, ns, agentLogFile)
})

It("should enable the bootstrap kubeconfig flow", func() {
It("should enable the bootstrap kubeconfig flow if the ~/.byoh/config does not exist", func() {
// start agent
var err error
output, _, err = runner.ExecByoDockerHost(byoHostContainer)
Expect(err).NotTo(HaveOccurred())
defer output.Close()
f := e2e.WriteDockerLog(output, agentLogFile)
defer func() {
@@ -561,7 +563,57 @@ var _ = Describe("Agent", func() {
return false
}, time.Second*2).Should(BeTrue())
})
It("should skip bootstrap kubeconfig flow if the ~/.byoh/config exists", func() {
shamsher31 marked this conversation as resolved.
Show resolved Hide resolved
cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
Expect(err).ShouldNot(HaveOccurred())
// create the directory to place the kubeconfig
execCommand, err := cli.ContainerExecCreate(ctx, byoHostContainer.ID, dockertypes.ExecConfig{
AttachStdin: false,
AttachStdout: true,
AttachStderr: true,
Cmd: []string{"sh", "-c", "mkdir ${HOME}/.byoh"},
})
Expect(err).ShouldNot(HaveOccurred())
err = cli.ContainerExecStart(ctx, execCommand.ID, dockertypes.ExecStartCheck{})
Expect(err).ShouldNot(HaveOccurred())
// copy the kubeconfig
execCommand, err = cli.ContainerExecCreate(ctx, byoHostContainer.ID, dockertypes.ExecConfig{
AttachStdin: false,
AttachStdout: true,
AttachStderr: true,
Cmd: []string{"sh", "-c", "cp /root/config ${HOME}/.byoh/"},
})
Expect(err).ShouldNot(HaveOccurred())
err = cli.ContainerExecStart(ctx, execCommand.ID, dockertypes.ExecStartCheck{})
Expect(err).ShouldNot(HaveOccurred())

// start agent
output, _, err = runner.ExecByoDockerHost(byoHostContainer)
Expect(err).NotTo(HaveOccurred())

defer output.Close()
f := e2e.WriteDockerLog(output, agentLogFile)
defer func() {
deferredErr := f.Close()
if deferredErr != nil {
e2e.Showf("error closing file %s: %v", agentLogFile, deferredErr)
}
}()
Consistently(func() (done bool) {
_, err := os.Stat(agentLogFile)
if err == nil {
data, err := os.ReadFile(agentLogFile)
if err == nil && strings.Contains(string(data), "\"msg\"=\"initiated bootstrap kubeconfig flow\"") {
return false
}
}
return true
}, time.Second*2).Should(BeTrue())
})
It("should not register the BYOHost with the management cluster", func() {
// start agent
_, _, err := runner.ExecByoDockerHost(byoHostContainer)
Expect(err).NotTo(HaveOccurred())
byoHostLookupKey := types.NamespacedName{Name: hostName, Namespace: ns.Name}
createdByoHost := &infrastructurev1beta1.ByoHost{}
Consistently(func() *infrastructurev1beta1.ByoHost {
@@ -573,6 +625,9 @@ var _ = Describe("Agent", func() {
}).Should(BeNil())
})
It("should create ByoHost CSR in the management cluster", func() {
// start agent
output, _, err := runner.ExecByoDockerHost(byoHostContainer)
Expect(err).NotTo(HaveOccurred())
defer output.Close()
f := e2e.WriteDockerLog(output, agentLogFile)
defer func() {
@@ -593,6 +648,9 @@ var _ = Describe("Agent", func() {
}, 10, 1).Should(Equal(fmt.Sprintf(registration.ByohCSRNameFormat, hostName)))
})
It("should persist private key", func() {
// start agent
output, _, err := runner.ExecByoDockerHost(byoHostContainer)
Expect(err).NotTo(HaveOccurred())
defer output.Close()
fAgent := e2e.WriteDockerLog(output, agentLogFile)
defer func() {
@@ -635,6 +693,9 @@ var _ = Describe("Agent", func() {
Expect(os.Remove(execLogFile)).ShouldNot(HaveOccurred())
})
It("should wait for the certificate to be issued", func() {
// start agent
output, _, err := runner.ExecByoDockerHost(byoHostContainer)
Expect(err).NotTo(HaveOccurred())
defer output.Close()
f := e2e.WriteDockerLog(output, agentLogFile)
defer func() {
@@ -655,6 +716,9 @@ var _ = Describe("Agent", func() {
}, time.Second*4).Should(BeTrue())
})
It("should create kubeconfig if the csr is approved", func() {
// start agent
output, _, err := runner.ExecByoDockerHost(byoHostContainer)
Expect(err).NotTo(HaveOccurred())
defer output.Close()
fAgent := e2e.WriteDockerLog(output, agentLogFile)
defer func() {
@@ -666,8 +730,8 @@ var _ = Describe("Agent", func() {

// Approve CSR
Eventually(func() (done bool) {
byohCSR, err := clientSet.CertificatesV1().CertificateSigningRequests().Get(ctx, fmt.Sprintf(registration.ByohCSRNameFormat, hostName), metav1.GetOptions{})
if err != nil {
byohCSR, kerr := clientSet.CertificatesV1().CertificateSigningRequests().Get(ctx, fmt.Sprintf(registration.ByohCSRNameFormat, hostName), metav1.GetOptions{})
if kerr != nil {
return false
}
byohCSR.Status.Conditions = append(byohCSR.Status.Conditions, certv1.CertificateSigningRequestCondition{
8 changes: 5 additions & 3 deletions agent/main.go
Original file line number Diff line number Diff line change
@@ -5,6 +5,7 @@ package main

import (
"context"
"errors"
"flag"
"fmt"
"os"
@@ -149,15 +150,16 @@ func main() {
logger.Error(err, "could not determine hostname")
return
}

_, err = os.Stat(registration.GetBYOHConfigPath())
shamsher31 marked this conversation as resolved.
Show resolved Hide resolved
// Enable bootstrap flow if --bootstrap-kubeconfig is provided
if bootstrapKubeConfig != "" {
// and config doesn't already exists in ~/.byoh/
if bootstrapKubeConfig != "" && errors.Is(err, os.ErrNotExist) {
if err = handleBootstrapFlow(logger, hostName); err != nil {
logger.Error(err, "bootstrap flow failed")
os.Exit(1)
}
}
// Handle restart flow
// Handle restart flow or if the ~/.byoh/config already exists
config, err := registration.LoadRESTClientConfig(registration.GetBYOHConfigPath())
if err != nil {
logger.Error(err, "client config load failed")
65 changes: 36 additions & 29 deletions test/e2e/docker_helper.go
Original file line number Diff line number Diff line change
@@ -191,35 +191,42 @@ func (r *ByoHostRunner) copyKubeconfig(config cpConfig, listopt types.ContainerL
re := regexp.MustCompile("server:.*")
kubeconfig = re.ReplaceAll(kubeconfig, []byte("server: https://127.0.0.1:"+r.Port))

// get the $HOME env variable to set the destination path for kubeconfig
execCommand, err := r.DockerClient.ContainerExecCreate(r.Context, containers[0].ID, types.ExecConfig{
AttachStdin: false,
AttachStdout: true,
AttachStderr: true,
Cmd: []string{"sh", "-c", "echo ${HOME}"},
})
Expect(err).ShouldNot(HaveOccurred())
resp, err := r.DockerClient.ContainerExecAttach(r.Context, execCommand.ID, types.ExecStartCheck{})
Expect(err).ShouldNot(HaveOccurred())
defer resp.Close()
homeDir, err := resp.Reader.ReadString('\n')
Expect(err).ShouldNot(HaveOccurred())
homeDir = strings.TrimSuffix(homeDir, "\n")
// create the directory to place the kubeconfig
execCommand, err = r.DockerClient.ContainerExecCreate(r.Context, containers[0].ID, types.ExecConfig{
AttachStdin: false,
AttachStdout: true,
AttachStderr: true,
Cmd: []string{"sh", "-c", "mkdir ${HOME}/.byoh"},
})
Expect(err).ShouldNot(HaveOccurred())
err = r.DockerClient.ContainerExecStart(r.Context, execCommand.ID, types.ExecStartCheck{})
Expect(err).ShouldNot(HaveOccurred())

Expect(os.WriteFile(tempKubeconfigPath, kubeconfig, 0644)).NotTo(HaveOccurred()) // nolint: gosec,gomnd
config.sourcePath = tempKubeconfigPath
// SplitAfterN used to remove the unwanted special characters in the homeDir
config.destPath = strings.SplitAfterN(strings.TrimSpace(homeDir)+"/.byoh/config", "/", 2)[1] // nolint: gomnd
// If the --bootstrap-kubeconfig is not provided, the tests will use
// kubeconfig placed in ~/.byoh/config
if r.CommandArgs["--bootstrap-kubeconfig"] == "" {
// get the $HOME env variable to set the destination path for kubeconfig
execCommand, err := r.DockerClient.ContainerExecCreate(r.Context, containers[0].ID, types.ExecConfig{
AttachStdin: false,
AttachStdout: true,
AttachStderr: true,
Cmd: []string{"sh", "-c", "echo ${HOME}"},
})
Expect(err).ShouldNot(HaveOccurred())
resp, err := r.DockerClient.ContainerExecAttach(r.Context, execCommand.ID, types.ExecStartCheck{})
Expect(err).ShouldNot(HaveOccurred())
defer resp.Close()
homeDir, err := resp.Reader.ReadString('\n')
Expect(err).ShouldNot(HaveOccurred())
homeDir = strings.TrimSuffix(homeDir, "\n")
// create the directory to place the kubeconfig
execCommand, err = r.DockerClient.ContainerExecCreate(r.Context, containers[0].ID, types.ExecConfig{
AttachStdin: false,
AttachStdout: true,
AttachStderr: true,
Cmd: []string{"sh", "-c", "mkdir ${HOME}/.byoh"},
})
Expect(err).ShouldNot(HaveOccurred())
err = r.DockerClient.ContainerExecStart(r.Context, execCommand.ID, types.ExecStartCheck{})
Expect(err).ShouldNot(HaveOccurred())

Expect(os.WriteFile(tempKubeconfigPath, kubeconfig, 0644)).NotTo(HaveOccurred()) // nolint: gosec,gomnd
config.sourcePath = tempKubeconfigPath
// SplitAfterN used to remove the unwanted special characters in the homeDir
config.destPath = strings.SplitAfterN(strings.TrimSpace(homeDir)+"/.byoh/config", "/", 2)[1] // nolint: gomnd
} else {
config.sourcePath = tempKubeconfigPath
config.destPath = r.CommandArgs["--bootstrap-kubeconfig"]
}
} else {
listopt.Filters.Add("name", r.clusterConName+"-control-plane")
containers, err := r.DockerClient.ContainerList(r.Context, listopt)