kubeconfig flag removed in the agent (#578)

* flag --kubeconfig removed

* Changes for fetching home dir for containers

* Create bootstrap kubeconfig for e2e tests

* update installer e2e

agent/host_agent_suite_test.go

@@ -41,7 +41,6 @@ var (
 	defaultByoMachineName = "my-byomachine"
 	agentLogFile          = "/tmp/agent-integration.log"
 	execLogFile           = "/tmp/agent-exec.log"
-	fakeKubeConfig        = "fake-kubeconfig-path"
 	fakeDownloadPath      = "fake-download-path"
 	fakeBootstrapSecret   = "fake-bootstrap-secret"
 	testEnv               *envtest.Environment
@@ -151,9 +150,8 @@ func setupTestInfra(ctx context.Context, hostname, kubeconfig string, namespace
 		ByoHostName:           hostname,
 		Port:                  testEnv.ControlPlane.APIServer.Port,
 		CommandArgs: map[string]string{
-			"--kubeconfig": "/mgmt.conf",
-			"--namespace":  namespace.Name,
-			"-v":           "1",
+			"--namespace": namespace.Name,
+			"-v":          "1",
 		},
 		KubeconfigFile: kubeconfig,
 	}

agent/host_agent_test.go

@@ -106,32 +106,6 @@ var _ = Describe("Agent", func() {
 				return false
 			}).Should(BeFalse())
 		})
-
-		It("should return an error when invalid kubeconfig is passed in", func() {
-
-			runner.CommandArgs["--kubeconfig"] = fakeKubeConfig
-			output, _, err := runner.ExecByoDockerHost(byoHostContainer)
-			Expect(err).NotTo(HaveOccurred())
-			defer output.Close()
-
-			f := e2e.WriteDockerLog(output, agentLogFile)
-			defer func() {
-				deferredErr := f.Close()
-				if deferredErr != nil {
-					e2e.Showf("error closing file %s: %v", agentLogFile, deferredErr)
-				}
-			}()
-			Eventually(func() (done bool) {
-				_, err := os.Stat(agentLogFile)
-				if err == nil {
-					data, err := os.ReadFile(agentLogFile)
-					if err == nil && strings.Contains(string(data), "\"msg\"=\"error getting kubeconfig\"") {
-						return true
-					}
-				}
-				return false
-			}).Should(BeTrue())
-		})
 	})
 
 	Context("When the host agent is able to connect to API Server", func() {
@@ -543,7 +517,7 @@ var _ = Describe("Agent", func() {
 			Expect(err).NotTo(HaveOccurred())
 
 			runner = setupTestInfra(ctx, hostName, getKubeConfig().Name(), ns)
-			runner.CommandArgs["--bootstrap-kubeconfig"] = "/mgmt.conf"
+			runner.CommandArgs["--bootstrap-kubeconfig"] = "/root/.byoh/config"
 			byoHostContainer, err = runner.SetupByoDockerHost()
 			Expect(err).NotTo(HaveOccurred())
 

agent/main.go

@@ -93,24 +93,13 @@ func setupflags() {
 
 	pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
 	hiddenFlags := []string{"log-flush-frequency", "alsologtostderr", "log-backtrace-at", "log-dir", "logtostderr", "stderrthreshold", "vmodule", "azure-container-registry-config",
-		"log_backtrace_at", "log_dir", "log_file", "log_file_max_size", "add_dir_header", "skip_headers", "skip_log_headers", "one_output"}
+		"log_backtrace_at", "log_dir", "log_file", "log_file_max_size", "add_dir_header", "skip_headers", "skip_log_headers", "one_output", "kubeconfig"}
 	for _, hiddenFlag := range hiddenFlags {
 		_ = pflag.CommandLine.MarkHidden(hiddenFlag)
 	}
 	feature.MutableGates.AddFlag(pflag.CommandLine)
 }
 
-func handleHostRegistration(k8sClient client.Client, hostName string, logger logr.Logger) (err error) {
-	registration.LocalHostRegistrar = &registration.HostRegistrar{K8sClient: k8sClient}
-	if bootstrapKubeConfig != "" {
-		logger.Info("bootstrap kubeconfig is provided, waiting for host to be registered by ByoHost Controller")
-	} else {
-		err := registration.LocalHostRegistrar.Register(hostName, namespace, labels)
-		return err
-	}
-	return nil
-}
-
 func setupTemplateParser() *cloudinit.TemplateParser {
 	var templateParser *cloudinit.TemplateParser
 	if registration.LocalHostRegistrar.ByoHostInfo.DefaultNetworkInterfaceName == "" {
@@ -168,23 +157,19 @@ func main() {
 			os.Exit(1)
 		}
 	}
-	// Handle kubeconfig flag first look in the byoh path for the kubeconfig
+	// Handle restart flow
 	config, err := registration.LoadRESTClientConfig(registration.GetBYOHConfigPath())
 	if err != nil {
 		logger.Error(err, "client config load failed")
-		// get the passed kubeconfig
-		config, err = ctrl.GetConfig()
-		if err != nil {
-			logger.Error(err, "error getting kubeconfig")
-			return
-		}
+		os.Exit(1)
 	}
 	k8sClient, err := client.New(config, client.Options{Scheme: scheme})
 	if err != nil {
 		logger.Error(err, "k8s client creation failed")
 		os.Exit(1)
 	}
-	err = handleHostRegistration(k8sClient, hostName, logger)
+	registration.LocalHostRegistrar = &registration.HostRegistrar{K8sClient: k8sClient}
+	err = registration.LocalHostRegistrar.Register(hostName, namespace, labels)
 	if err != nil {
 		logger.Error(err, "error registering host %s registration in namespace %s", hostName, namespace)
 		return

config/rbac/kustomization.yaml

@@ -9,6 +9,12 @@ resources:
 - role_binding.yaml
 - leader_election_role.yaml
 - leader_election_role_binding.yaml
+- byohost_editor_role.yaml
+- byohost_editor_clusterrolebinding.yaml
+- byoh_csr_creator_clusterrole.yaml
+- byoh_csr_creator_clusterrolebinding.yaml
+- secret_reader_clusterrole.yaml
+- secret_reader_clusterrolebinding.yaml
 # Comment the following 4 lines if you want to disable
 # the auth proxy (https://github.com/brancz/kube-rbac-proxy)
 # which protects your /metrics endpoint.

config/rbac/role.yaml

@@ -42,11 +42,21 @@ rules:
   - certificates.k8s.io
   resources:
   - certificatesigningrequests
+  - certificatesigningrequests/approval
   verbs:
   - create
+  - update
   - get
   - list
   - watch
+- apiGroups:
+  - certificates.k8s.io
+  resources:
+  - signers
+  resourceNames:
+  - "kubernetes.io/kube-apiserver-client"  
+  verbs:
+  - approve
 - apiGroups:
   - cluster.x-k8s.io
   resources:

config/rbac/secret_reader_clusterrole.yaml

@@ -0,0 +1,12 @@
+# permissions for end users to view byoclusters.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: secret-reader-clusterrole
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get

config/rbac/secret_reader_clusterrolebinding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: secret-reader-clusterrole-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: secret-reader-clusterrole
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: Group
+  name: byoh:hosts

controllers/infrastructure/byoadmission_controller.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 
 	certv1 "k8s.io/api/certificates/v1"
+	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	clientset "k8s.io/client-go/kubernetes"
@@ -57,6 +58,7 @@ func (r *ByoAdmissionReconciler) Reconcile(ctx context.Context, req ctrl.Request
 	// Update the CSR to the "Approved" condition
 	csr.Status.Conditions = append(csr.Status.Conditions, certv1.CertificateSigningRequestCondition{
 		Type:   certv1.CertificateApproved,
+		Status: corev1.ConditionTrue,
 		Reason: "Approved by ByoAdmission Controller",
 	})
 

controllers/infrastructure/byoadmission_controller_test.go

@@ -10,6 +10,7 @@ import (
 	. "github.com/onsi/gomega"
 	"github.com/vmware-tanzu/cluster-api-provider-bringyourownhost/test/builder"
 	certv1 "k8s.io/api/certificates/v1"
+	corev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
@@ -55,6 +56,7 @@ var _ = Describe("Controllers/ByoadmissionController", func() {
 			Expect(updateByohCSR.Status.Conditions).Should(ContainElement(certv1.CertificateSigningRequestCondition{
 				Type:   certv1.CertificateApproved,
 				Reason: "Approved by ByoAdmission Controller",
+				Status: corev1.ConditionTrue,
 			}))
 		})
 

test/e2e/byohost_reuse_test.go

@@ -76,9 +76,9 @@ var _ = Describe("When BYO Host rejoins the capacity pool", func() {
 			NetworkInterface:      "kind",
 			bootstrapClusterProxy: bootstrapClusterProxy,
 			CommandArgs: map[string]string{
-				"--kubeconfig": "/mgmt.conf",
-				"--namespace":  namespace.Name,
-				"--v":          "1",
+				"--bootstrap-kubeconfig": "/bootstrap.conf",
+				"--namespace":            namespace.Name,
+				"--v":                    "1",
 			},
 		}
 

test/e2e/data/infrastructure-provider-byoh/v1beta1/templates/e2e/installer/cluster-template.yaml

@@ -189,7 +189,6 @@ metadata:
   name: ${CLUSTER_NAME}
 spec:
   bundleLookupBaseRegistry: projects.registry.vmware.com/cluster_api_provider_bringyourownhost
-  bundleLookupTag: ${BUNDLE_LOOKUP_TAG}
   controlPlaneEndpoint:
     host: ${CONTROL_PLANE_ENDPOINT_IP}
     port: 6443

test/e2e/data/infrastructure-provider-byoh/v1beta1/templates/e2e/installer/cluster-with-kcp.yaml

@@ -28,7 +28,6 @@ metadata:
   name: ${CLUSTER_NAME}
 spec:
   bundleLookupBaseRegistry: projects.registry.vmware.com/cluster_api_provider_bringyourownhost
-  bundleLookupTag: ${BUNDLE_LOOKUP_TAG}
   controlPlaneEndpoint:
     host: ${CONTROL_PLANE_ENDPOINT_IP}
     port: 6443

test/e2e/docker_helper.go

@@ -10,6 +10,7 @@ import (
 	"path/filepath"
 	"regexp"
 	"strings"
+	"time"
 
 	"github.com/docker/cli/cli/command"
 	"github.com/docker/docker/api/types"
@@ -21,12 +22,15 @@ import (
 	"github.com/docker/docker/pkg/system"
 	. "github.com/onsi/gomega" // nolint: stylecheck
 	"github.com/pkg/errors"
+	"github.com/vmware-tanzu/cluster-api-provider-bringyourownhost/test/utils/bootstraptoken"
 	"sigs.k8s.io/cluster-api/test/framework"
 )
 
 const (
-	kindImage          = "byoh/node:e2e"
-	tempKubeconfigPath = "/tmp/mgmt.conf"
+	kindImage           = "byoh/node:e2e"
+	tempKubeconfigPath  = "/tmp/mgmt.conf"
+	bootstrapKubeconfig = "/tmp/boostrap-kubeconfig"
+	ttl                 = time.Minute * 30
 )
 
 type cpConfig struct {
@@ -186,25 +190,55 @@ func (r *ByoHostRunner) copyKubeconfig(config cpConfig, listopt types.ContainerL
 
 		re := regexp.MustCompile("server:.*")
 		kubeconfig = re.ReplaceAll(kubeconfig, []byte("server: https://127.0.0.1:"+r.Port))
+
+		// get the $HOME env variable to set the destination path for kubeconfig
+		execCommand, err := r.DockerClient.ContainerExecCreate(r.Context, containers[0].ID, types.ExecConfig{
+			AttachStdin:  false,
+			AttachStdout: true,
+			AttachStderr: true,
+			Cmd:          []string{"sh", "-c", "echo ${HOME}"},
+		})
+		Expect(err).ShouldNot(HaveOccurred())
+		resp, err := r.DockerClient.ContainerExecAttach(r.Context, execCommand.ID, types.ExecStartCheck{})
+		Expect(err).ShouldNot(HaveOccurred())
+		defer resp.Close()
+		homeDir, err := resp.Reader.ReadString('\n')
+		Expect(err).ShouldNot(HaveOccurred())
+		homeDir = strings.TrimSuffix(homeDir, "\n")
+		// create the directory to place the kubeconfig
+		execCommand, err = r.DockerClient.ContainerExecCreate(r.Context, containers[0].ID, types.ExecConfig{
+			AttachStdin:  false,
+			AttachStdout: true,
+			AttachStderr: true,
+			Cmd:          []string{"sh", "-c", "mkdir ${HOME}/.byoh"},
+		})
+		Expect(err).ShouldNot(HaveOccurred())
+		err = r.DockerClient.ContainerExecStart(r.Context, execCommand.ID, types.ExecStartCheck{})
+		Expect(err).ShouldNot(HaveOccurred())
+
+		Expect(os.WriteFile(tempKubeconfigPath, kubeconfig, 0644)).NotTo(HaveOccurred()) // nolint: gosec,gomnd
+		config.sourcePath = tempKubeconfigPath
+		// SplitAfterN used to remove the unwanted special characters in the homeDir
+		config.destPath = strings.SplitAfterN(strings.TrimSpace(homeDir)+"/.byoh/config", "/", 2)[1] // nolint: gomnd
 	} else {
 		listopt.Filters.Add("name", r.clusterConName+"-control-plane")
 		containers, err := r.DockerClient.ContainerList(r.Context, listopt)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(len(containers)).To(Equal(1))
-
 		profile, err := r.DockerClient.ContainerInspect(r.Context, containers[0].ID)
 		Expect(err).NotTo(HaveOccurred())
-
 		kubeconfig, err = os.ReadFile(r.bootstrapClusterProxy.GetKubeconfigPath())
 		Expect(err).NotTo(HaveOccurred())
-
 		re := regexp.MustCompile("server:.*")
 		kubeconfig = re.ReplaceAll(kubeconfig, []byte("server: https://"+profile.NetworkSettings.Networks[r.NetworkInterface].IPAddress+":6443"))
-	}
-	Expect(os.WriteFile(tempKubeconfigPath, kubeconfig, 0644)).NotTo(HaveOccurred()) // nolint: gosec,gomnd
+		config.destPath = r.CommandArgs["--bootstrap-kubeconfig"]
+		Expect(os.WriteFile(tempKubeconfigPath, kubeconfig, 0644)).NotTo(HaveOccurred()) // nolint: gosec,gomnd
 
-	config.sourcePath = tempKubeconfigPath
-	config.destPath = r.CommandArgs["--kubeconfig"]
+		// Create Bootstrap kubeconfig
+		err = bootstraptoken.CreateBootstrapKubeConfig(tempKubeconfigPath, bootstrapKubeconfig, ttl)
+		Expect(err).ShouldNot(HaveOccurred())
+		config.sourcePath = bootstrapKubeconfig
+	}
 	err := copyToContainer(r.Context, r.DockerClient, config)
 	return err
 }

test/e2e/e2e_installer_test.go

@@ -72,7 +72,7 @@ var _ = Describe("When BYOH joins existing cluster [Installer]", func() {
 			NetworkInterface:      "kind",
 			bootstrapClusterProxy: bootstrapClusterProxy,
 			CommandArgs: map[string]string{
-				"--kubeconfig":               "/mgmt.conf",
+				"--bootstrap-kubeconfig":     "/bootstrap.conf",
 				"--namespace":                namespace.Name,
 				"--v":                        "1",
 				"--use-installer-controller": "true",

test/e2e/e2e_test.go

@@ -72,9 +72,9 @@ var _ = Describe("When BYOH joins existing cluster [PR-Blocking]", func() {
 			NetworkInterface:      "kind",
 			bootstrapClusterProxy: bootstrapClusterProxy,
 			CommandArgs: map[string]string{
-				"--kubeconfig": "/mgmt.conf",
-				"--namespace":  namespace.Name,
-				"--v":          "1",
+				"--bootstrap-kubeconfig": "/bootstrap.conf",
+				"--namespace":            namespace.Name,
+				"--v":                    "1",
 			},
 		}
 

test/e2e/md_scale_test.go

@@ -74,9 +74,9 @@ var _ = Describe("When testing MachineDeployment scale out/in", func() {
 				NetworkInterface:      "kind",
 				bootstrapClusterProxy: bootstrapClusterProxy,
 				CommandArgs: map[string]string{
-					"--kubeconfig": "/mgmt.conf",
-					"--namespace":  namespace.Name,
-					"--v":          "1",
+					"--bootstrap-kubeconfig": "/bootstrap.conf",
+					"--namespace":            namespace.Name,
+					"--v":                    "1",
 				},
 			}
 			byohost, err := runner.SetupByoDockerHost()