From 46412e6c3f75a05d51a8d6b074ff030fb8987e54 Mon Sep 17 00:00:00 2001 From: Anusha Hegde Date: Fri, 10 Jun 2022 05:31:49 +0000 Subject: [PATCH] BootstrapKubeConfig controller implementation --- .../v1beta1/bootstrapkubeconfig_types.go | 3 +- .../v1beta1/zz_generated.deepcopy.go | 7 +- common/utils.go | 18 +++++ ...cluster.x-k8s.io_bootstrapkubeconfigs.yaml | 35 +-------- .../bootstrapkubeconfig_controller.go | 73 ++++++++++++++++++- .../bootstrapkubeconfig_controller_test.go | 13 +++- controllers/infrastructure/suite_test.go | 1 + go.mod | 10 +-- main.go | 2 + 9 files changed, 112 insertions(+), 50 deletions(-) diff --git a/apis/infrastructure/v1beta1/bootstrapkubeconfig_types.go b/apis/infrastructure/v1beta1/bootstrapkubeconfig_types.go index 53c50aebe..f3c75a9bf 100644 --- a/apis/infrastructure/v1beta1/bootstrapkubeconfig_types.go +++ b/apis/infrastructure/v1beta1/bootstrapkubeconfig_types.go @@ -4,7 +4,6 @@ package v1beta1 import ( - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -25,7 +24,7 @@ type BootstrapKubeconfigStatus struct { // BootstrapKubeconfigData is an optional reference to a bootstrap kubeconfig info // for starting the host registration process // +optional - BootstrapKubeconfigData *corev1.ObjectReference `json:"bootstrapKubeconfigData,omitempty"` + BootstrapKubeconfigData string `json:"bootstrapKubeconfigData,omitempty"` } //+kubebuilder:object:root=true diff --git a/apis/infrastructure/v1beta1/zz_generated.deepcopy.go b/apis/infrastructure/v1beta1/zz_generated.deepcopy.go index 099de227e..f315d8724 100644 --- a/apis/infrastructure/v1beta1/zz_generated.deepcopy.go +++ b/apis/infrastructure/v1beta1/zz_generated.deepcopy.go @@ -36,7 +36,7 @@ func (in *BootstrapKubeconfig) DeepCopyInto(out *BootstrapKubeconfig) { out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) out.Spec = in.Spec - in.Status.DeepCopyInto(&out.Status) + out.Status = in.Status } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BootstrapKubeconfig. @@ -107,11 +107,6 @@ func (in *BootstrapKubeconfigSpec) DeepCopy() *BootstrapKubeconfigSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BootstrapKubeconfigStatus) DeepCopyInto(out *BootstrapKubeconfigStatus) { *out = *in - if in.BootstrapKubeconfigData != nil { - in, out := &in.BootstrapKubeconfigData, &out.BootstrapKubeconfigData - *out = new(v1.ObjectReference) - **out = **in - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BootstrapKubeconfigStatus. diff --git a/common/utils.go b/common/utils.go index 40dc92b2f..3d46c2b16 100644 --- a/common/utils.go +++ b/common/utils.go @@ -6,9 +6,12 @@ package common import ( "bytes" "compress/gzip" + "encoding/base64" + "encoding/json" "io" "os" "path/filepath" + "strings" "github.com/pkg/errors" ) @@ -66,3 +69,18 @@ func RemoveGlob(path string) error { } return nil } + +func EncodeToBase64(v interface{}) (string, error) { + var buf bytes.Buffer + encoder := base64.NewEncoder(base64.StdEncoding, &buf) + err := json.NewEncoder(encoder).Encode(v) + if err != nil { + return "", err + } + encoder.Close() + return buf.String(), nil +} + +func DecodeFromBase64(v interface{}, enc string) error { + return json.NewDecoder(base64.NewDecoder(base64.StdEncoding, strings.NewReader(enc))).Decode(v) +} diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_bootstrapkubeconfigs.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_bootstrapkubeconfigs.yaml index 57d421bad..9faea11c0 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_bootstrapkubeconfigs.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_bootstrapkubeconfigs.yaml @@ -48,40 +48,7 @@ spec: bootstrapKubeconfigData: description: BootstrapKubeconfigData is an optional reference to a bootstrap kubeconfig info for starting the host registration process - properties: - apiVersion: - description: API version of the referent. - type: string - fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' - type: string - kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' - type: string - namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' - type: string - resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' - type: string - uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' - type: string - type: object + type: string type: object type: object served: true diff --git a/controllers/infrastructure/bootstrapkubeconfig_controller.go b/controllers/infrastructure/bootstrapkubeconfig_controller.go index d91cfcfdb..6f8b645a1 100644 --- a/controllers/infrastructure/bootstrapkubeconfig_controller.go +++ b/controllers/infrastructure/bootstrapkubeconfig_controller.go @@ -5,10 +5,19 @@ package controllers import ( "context" + "fmt" infrastructurev1beta1 "github.com/vmware-tanzu/cluster-api-provider-bringyourownhost/apis/infrastructure/v1beta1" + "github.com/vmware-tanzu/cluster-api-provider-bringyourownhost/common" + v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/rest" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + bootstrapapi "k8s.io/cluster-bootstrap/token/api" + bootstraputil "k8s.io/cluster-bootstrap/token/util" + "sigs.k8s.io/cluster-api/util/patch" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" @@ -18,6 +27,7 @@ import ( type BootstrapKubeconfigReconciler struct { client.Client Scheme *runtime.Scheme + *rest.Config } //+kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=bootstrapkubeconfigs,verbs=get;list;watch;create;update;patch;delete @@ -43,8 +53,69 @@ func (r *BootstrapKubeconfigReconciler) Reconcile(ctx context.Context, req ctrl. } return ctrl.Result{}, err } + tokenStr, err := bootstraputil.GenerateBootstrapToken() + if err != nil { + return ctrl.Result{}, nil + } + substrs := bootstraputil.BootstrapTokenRegexp.FindStringSubmatch(tokenStr) + tokenID := substrs[1] + tokenSecret := substrs[2] + + data := map[string][]byte{ + bootstrapapi.BootstrapTokenIDKey: []byte(tokenID), + bootstrapapi.BootstrapTokenSecretKey: []byte(tokenSecret), + } + description := "BYOH controller generated bootstrap token" + data["Description"] = []byte(description) + // for _, usage := range token.Usages { + // data[bootstrapapi.BootstrapTokenUsagePrefix+usage] = []byte("true") + // } + data[bootstrapapi.BootstrapTokenExtraGroupsKey] = []byte("system:bootstrappers:byoh") + + bootstrapSecret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: bootstraputil.BootstrapTokenSecretName(tokenID), + Namespace: metav1.NamespaceSystem, + }, + Type: bootstrapapi.SecretTypeBootstrapToken, + Data: data, + } + + // TODO: check if secret with name already exists + // if exists, maybe update the secret? + err = r.Client.Create(ctx, bootstrapSecret) + if err != nil { + return ctrl.Result{}, err + } + + // Build resulting kubeconfig. + kubeconfigData := clientcmdapi.Config{ + // Define a cluster stanza based on the bootstrap kubeconfig. + Clusters: map[string]*clientcmdapi.Cluster{"default-cluster": { + Server: r.Config.Host, + InsecureSkipTLSVerify: r.Config.Insecure, + CertificateAuthorityData: r.Config.CAData, + }}, + // Define auth based on the obtained client cert. + AuthInfos: map[string]*clientcmdapi.AuthInfo{"default-auth": { + Token: fmt.Sprintf(tokenID + "." + tokenSecret), + }}, + // Define a context that connects the auth info and cluster, and set it as the default + Contexts: map[string]*clientcmdapi.Context{"default-context": { + Cluster: "default-cluster", + AuthInfo: "default-auth", + Namespace: "default", + }}, + CurrentContext: "default-context", + } + encodedKubeConfig, err := common.EncodeToBase64(kubeconfigData) + helper, err := patch.NewHelper(bootstrapKubeconfig, r.Client) + if err != nil { + return ctrl.Result{}, err + } + bootstrapKubeconfig.Status.BootstrapKubeconfigData = encodedKubeConfig - return ctrl.Result{}, nil + return ctrl.Result{}, helper.Patch(ctx, bootstrapKubeconfig) } // SetupWithManager sets up the controller with the Manager. diff --git a/controllers/infrastructure/bootstrapkubeconfig_controller_test.go b/controllers/infrastructure/bootstrapkubeconfig_controller_test.go index 6ea3ac545..17b3a3423 100644 --- a/controllers/infrastructure/bootstrapkubeconfig_controller_test.go +++ b/controllers/infrastructure/bootstrapkubeconfig_controller_test.go @@ -9,9 +9,11 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" infrav1 "github.com/vmware-tanzu/cluster-api-provider-bringyourownhost/apis/infrastructure/v1beta1" + "github.com/vmware-tanzu/cluster-api-provider-bringyourownhost/common" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) @@ -52,7 +54,7 @@ var _ = Describe("Controllers/BoottrapKubeconfigController", func() { Expect(err).NotTo(HaveOccurred()) }) - Context("When BootstrapKubeconf CRD is created", func() { + Context("When BootstrapKubeconfig CRD is created", func() { It("should generate the bootstrap kubeconfig data", func() { _, err := bootstrapKubeconfigReconciler.Reconcile(ctx, reconcile.Request{ NamespacedName: bootstrapKubeconfigLookupKey}) @@ -61,7 +63,14 @@ var _ = Describe("Controllers/BoottrapKubeconfigController", func() { createdBootstrapKubeconfig := &infrav1.BootstrapKubeconfig{} err = k8sClientUncached.Get(ctx, bootstrapKubeconfigLookupKey, createdBootstrapKubeconfig) Expect(err).ToNot(HaveOccurred()) - Expect(createdBootstrapKubeconfig.Status.BootstrapKubeconfigData).Should(Equal("")) + Expect(createdBootstrapKubeconfig.Status.BootstrapKubeconfigData).ShouldNot(BeEmpty()) + + var config *clientcmdapi.Config + + // should be able to decode into a Config struct + err = common.DecodeFromBase64(&config, createdBootstrapKubeconfig.Status.BootstrapKubeconfigData) + Expect(err).ToNot(HaveOccurred()) + }) }) }) diff --git a/controllers/infrastructure/suite_test.go b/controllers/infrastructure/suite_test.go index 81ba14a8e..6810c99bc 100644 --- a/controllers/infrastructure/suite_test.go +++ b/controllers/infrastructure/suite_test.go @@ -153,6 +153,7 @@ var _ = BeforeSuite(func() { bootstrapKubeconfigReconciler = &controllers.BootstrapKubeconfigReconciler{ Client: k8sManager.GetClient(), + Config: cfg, } err = bootstrapKubeconfigReconciler.SetupWithManager(k8sManager) Expect(err).NotTo(HaveOccurred()) diff --git a/go.mod b/go.mod index 7cecb1bd5..403b0d61b 100644 --- a/go.mod +++ b/go.mod @@ -17,10 +17,11 @@ require ( github.com/onsi/gomega v1.19.0 github.com/pkg/errors v0.9.1 github.com/spf13/pflag v1.0.5 - k8s.io/api v0.24.1 - k8s.io/apimachinery v0.24.1 - k8s.io/client-go v0.24.1 - k8s.io/component-base v0.24.1 + k8s.io/api v0.24.0 + k8s.io/apimachinery v0.24.0 + k8s.io/client-go v0.24.0 + k8s.io/cluster-bootstrap v0.23.0 + k8s.io/component-base v0.24.0 k8s.io/klog/v2 v2.60.1 k8s.io/kubectl v0.24.1 k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 @@ -183,7 +184,6 @@ require ( k8s.io/apiextensions-apiserver v0.23.5 // indirect k8s.io/apiserver v0.23.5 // indirect k8s.io/cloud-provider v0.21.0 // indirect - k8s.io/cluster-bootstrap v0.23.0 // indirect k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42 // indirect k8s.io/legacy-cloud-providers v0.21.0 // indirect sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect diff --git a/main.go b/main.go index 7fe05631d..6f959043d 100644 --- a/main.go +++ b/main.go @@ -92,6 +92,7 @@ func main() { setupLog.Error(err, "unable to create cluster cache tracker") os.Exit(1) } + if err = (&remote.ClusterCacheReconciler{ Client: mgr.GetClient(), Log: ctrl.Log.WithName("remote").WithName("ClusterCacheReconciler"), @@ -151,6 +152,7 @@ func main() { if err = (&infrastructurecontrollers.BootstrapKubeconfigReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), + Config: mgr.GetConfig(), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "BootstrapKubeconfig") os.Exit(1)