From 9c0da69a71b3fc3200e1490d21b01e47e50e29dc Mon Sep 17 00:00:00 2001 From: Paolo Dettori Date: Fri, 7 Jul 2023 21:13:53 -0400 Subject: [PATCH] add support for vcluster control planes Signed-off-by: Paolo Dettori --- cmd/kflex/create/create.go | 22 ++++-- .../controller/controlplane_controller.go | 4 + pkg/certs/kconfig_gen.go | 3 +- pkg/kubeconfig/kubeconfig.go | 12 ++- pkg/reconcilers/k8s/reconciler.go | 2 +- pkg/reconcilers/ocm/chart.go | 16 ---- pkg/reconcilers/ocm/reconciler.go | 2 +- pkg/reconcilers/shared/ingress.go | 8 +- pkg/reconcilers/vcluster/chart.go | 79 +++++++++++++++++++ pkg/reconcilers/vcluster/reconciler.go | 66 ++++++++++++++++ pkg/util/util.go | 38 ++++++--- 11 files changed, 212 insertions(+), 40 deletions(-) create mode 100644 pkg/reconcilers/vcluster/chart.go create mode 100644 pkg/reconcilers/vcluster/reconciler.go diff --git a/cmd/kflex/create/create.go b/cmd/kflex/create/create.go index b0b9432..b21311d 100644 --- a/cmd/kflex/create/create.go +++ b/cmd/kflex/create/create.go @@ -59,12 +59,22 @@ func (c *CPCreate) Create(controlPlaneType, backendType string) { util.PrintStatus("Waiting for API server to become ready...", done, &wg) kubeconfig.WatchForSecretCreation(clientset, c.Name, util.GetKubeconfSecretNameByControlPlaneType(controlPlaneType)) - if err := util.WaitForDeploymentReady(clientset, - util.GetAPIServerDeploymentNameByControlPlaneType(controlPlaneType), - util.GenerateNamespaceFromControlPlaneName(cp.Name)); err != nil { - - fmt.Fprintf(os.Stderr, "Error waiting for deployment to become ready: %v\n", err) - os.Exit(1) + if controlPlaneType == string(tenancyv1alpha1.ControlPlaneTypeVCluster) { + if err := util.WaitForStatefulSetReady(clientset, + util.GetAPIServerDeploymentNameByControlPlaneType(controlPlaneType), + util.GenerateNamespaceFromControlPlaneName(cp.Name)); err != nil { + + fmt.Fprintf(os.Stderr, "Error waiting for stateful set to become ready: %v\n", err) + os.Exit(1) + } + } else { + if err := util.WaitForDeploymentReady(clientset, + util.GetAPIServerDeploymentNameByControlPlaneType(controlPlaneType), + util.GenerateNamespaceFromControlPlaneName(cp.Name)); err != nil { + + fmt.Fprintf(os.Stderr, "Error waiting for deployment to become ready: %v\n", err) + os.Exit(1) + } } done <- true diff --git a/internal/controller/controlplane_controller.go b/internal/controller/controlplane_controller.go index cbeb72b..2e3a1c7 100644 --- a/internal/controller/controlplane_controller.go +++ b/internal/controller/controlplane_controller.go @@ -32,6 +32,7 @@ import ( tenancyv1alpha1 "github.com/kubestellar/kubeflex/api/v1alpha1" "github.com/kubestellar/kubeflex/pkg/reconcilers/k8s" "github.com/kubestellar/kubeflex/pkg/reconcilers/ocm" + "github.com/kubestellar/kubeflex/pkg/reconcilers/vcluster" "github.com/kubestellar/kubeflex/pkg/util" ) @@ -94,6 +95,9 @@ func (r *ControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Request case tenancyv1alpha1.ControlPlaneTypeOCM: reconciler := ocm.New(r.Client, r.Scheme) return reconciler.Reconcile(ctx, hcp) + case tenancyv1alpha1.ControlPlaneTypeVCluster: + reconciler := vcluster.New(r.Client, r.Scheme) + return reconciler.Reconcile(ctx, hcp) default: return ctrl.Result{}, fmt.Errorf("unsupported control plane type: %s", hcp.Spec.Type) } diff --git a/pkg/certs/kconfig_gen.go b/pkg/certs/kconfig_gen.go index bf10968..9a68ff1 100644 --- a/pkg/certs/kconfig_gen.go +++ b/pkg/certs/kconfig_gen.go @@ -44,7 +44,6 @@ const ( Organization = "system:masters" ContrCMCN = "system:kube-controller-manager" CMConfSecret = "cm-kubeconfig" - ConfSecretKey = "kubeconfig" ) type ConfigGen struct { @@ -85,7 +84,7 @@ func (c *ConfigGen) genSecretManifest(ctx context.Context, conf []byte) *v1.Secr }, Type: v1.SecretTypeOpaque, Data: map[string][]byte{ - ConfSecretKey: conf, + util.KubeconfigSecretKeyDefault: conf, }, } } diff --git a/pkg/kubeconfig/kubeconfig.go b/pkg/kubeconfig/kubeconfig.go index 5b05487..3445f62 100644 --- a/pkg/kubeconfig/kubeconfig.go +++ b/pkg/kubeconfig/kubeconfig.go @@ -62,7 +62,8 @@ func loadControlPlaneKubeconfig(ctx context.Context, client kubernetes.Clientset return nil, err } - return clientcmd.Load(ks.Data[certs.ConfSecretKey]) + key := util.GetKubeconfSecretKeyNameByControlPlaneType(controlPlaneType) + return clientcmd.Load(ks.Data[key]) } func LoadKubeconfig(ctx context.Context) (*clientcmdapi.Config, error) { @@ -117,6 +118,15 @@ func adjustConfigKeys(config *clientcmdapi.Config, cpName, controlPlaneType stri Cluster: certs.GenerateClusterName(cpName), AuthInfo: certs.GenerateAuthInfoAdminName(cpName), } + case string(tenancyv1alpha1.ControlPlaneTypeVCluster): + renameKey(config.Clusters, "my-vcluster", certs.GenerateClusterName(cpName)) + renameKey(config.AuthInfos, "my-vcluster", certs.GenerateAuthInfoAdminName(cpName)) + renameKey(config.Contexts, "my-vcluster", certs.GenerateContextName(cpName)) + config.CurrentContext = certs.GenerateContextName(cpName) + config.Contexts[certs.GenerateContextName(cpName)] = &clientcmdapi.Context{ + Cluster: certs.GenerateClusterName(cpName), + AuthInfo: certs.GenerateAuthInfoAdminName(cpName), + } default: return } diff --git a/pkg/reconcilers/k8s/reconciler.go b/pkg/reconcilers/k8s/reconciler.go index 5e2617d..4c7ae4f 100644 --- a/pkg/reconcilers/k8s/reconciler.go +++ b/pkg/reconcilers/k8s/reconciler.go @@ -77,7 +77,7 @@ func (r *K8sReconciler) Reconcile(ctx context.Context, hcp *tenancyv1alpha1.Cont return r.UpdateStatusForSyncingError(hcp, err) } - if err = r.ReconcileAPIServerIngress(ctx, hcp, ""); err != nil { + if err = r.ReconcileAPIServerIngress(ctx, hcp, "", shared.SecurePort); err != nil { return r.UpdateStatusForSyncingError(hcp, err) } diff --git a/pkg/reconcilers/ocm/chart.go b/pkg/reconcilers/ocm/chart.go index 7c41797..499b5b2 100644 --- a/pkg/reconcilers/ocm/chart.go +++ b/pkg/reconcilers/ocm/chart.go @@ -26,18 +26,6 @@ import ( "github.com/kubestellar/kubeflex/pkg/util" ) -// helm install -n cp3-system multicluster-controlplane \ -// oci://quay.io/pdettori/multicluster-controlplane-chart \ -// --version 0.1.0 \ -// --create-namespace \ -// --set image=quay.io/pdettori/multicluster-controlplane:latest \ -// --set route.enabled=false \ -// --set apiserver.externalHostname=cp3.localtest.me \ -// --set apiserver.internalHostname=kubeflex-control-plane \ -// --set apiserver.port=9443 \ -// --set nodeport.enabled=true \ -// --set nodeport.port=30443 - const ( URL = "oci://quay.io/pdettori/multicluster-controlplane-chart:0.1.0" RepoName = "multicluster-controlplane" @@ -53,10 +41,6 @@ var ( "apiserver.port=9443", "nodeport.enabled=false", } - - Args = map[string]string{ - "set": "image=quay.io/pdettori/multicluster-controlplane:latest,route.enabled=false", - } ) func (r *OCMReconciler) ReconcileChart(ctx context.Context, hcp *tenancyv1alpha1.ControlPlane) error { diff --git a/pkg/reconcilers/ocm/reconciler.go b/pkg/reconcilers/ocm/reconciler.go index 1b9f862..0de67c6 100644 --- a/pkg/reconcilers/ocm/reconciler.go +++ b/pkg/reconcilers/ocm/reconciler.go @@ -57,7 +57,7 @@ func (r *OCMReconciler) Reconcile(ctx context.Context, hcp *tenancyv1alpha1.Cont return r.UpdateStatusForSyncingError(hcp, err) } - if err := r.ReconcileAPIServerIngress(ctx, hcp, ServiceName); err != nil { + if err := r.ReconcileAPIServerIngress(ctx, hcp, ServiceName, shared.SecurePort); err != nil { return r.UpdateStatusForSyncingError(hcp, err) } diff --git a/pkg/reconcilers/shared/ingress.go b/pkg/reconcilers/shared/ingress.go index 56fb96f..efe0f27 100644 --- a/pkg/reconcilers/shared/ingress.go +++ b/pkg/reconcilers/shared/ingress.go @@ -39,7 +39,7 @@ var ( pathTypePrefix = networkingv1.PathTypePrefix ) -func (r *BaseReconciler) ReconcileAPIServerIngress(ctx context.Context, hcp *tenancyv1alpha1.ControlPlane, svcName string) error { +func (r *BaseReconciler) ReconcileAPIServerIngress(ctx context.Context, hcp *tenancyv1alpha1.ControlPlane, svcName string, svcPort int) error { _ = clog.FromContext(ctx) namespace := util.GenerateNamespaceFromControlPlaneName(hcp.Name) @@ -58,7 +58,7 @@ func (r *BaseReconciler) ReconcileAPIServerIngress(ctx context.Context, hcp *ten err := r.Client.Get(context.TODO(), client.ObjectKeyFromObject(ingress), ingress, &client.GetOptions{}) if err != nil { if apierrors.IsNotFound(err) { - ingress = generateAPIServerIngress(hcp.Name, svcName, namespace) + ingress = generateAPIServerIngress(hcp.Name, svcName, namespace, svcPort) if err := controllerutil.SetControllerReference(hcp, ingress, r.Scheme); err != nil { return nil } @@ -71,7 +71,7 @@ func (r *BaseReconciler) ReconcileAPIServerIngress(ctx context.Context, hcp *ten return nil } -func generateAPIServerIngress(name, svcName, namespace string) *networkingv1.Ingress { +func generateAPIServerIngress(name, svcName, namespace string, svcPort int) *networkingv1.Ingress { return &networkingv1.Ingress{ TypeMeta: metav1.TypeMeta{ Kind: "Ingress", @@ -99,7 +99,7 @@ func generateAPIServerIngress(name, svcName, namespace string) *networkingv1.Ing Service: &networkingv1.IngressServiceBackend{ Name: svcName, Port: networkingv1.ServiceBackendPort{ - Number: SecurePort, + Number: int32(svcPort), }, }, }, diff --git a/pkg/reconcilers/vcluster/chart.go b/pkg/reconcilers/vcluster/chart.go new file mode 100644 index 0000000..6f6318b --- /dev/null +++ b/pkg/reconcilers/vcluster/chart.go @@ -0,0 +1,79 @@ +/* +Copyright 2023 The KubeStellar Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vcluster + +import ( + "context" + "fmt" + "strings" + + tenancyv1alpha1 "github.com/kubestellar/kubeflex/api/v1alpha1" + "github.com/kubestellar/kubeflex/pkg/helm" + "github.com/kubestellar/kubeflex/pkg/util" +) + +// helm upgrade --install vcluster \ +// --set vcluster.image=rancher/k3s:v1.27.2-k3s1 \ +// --repo https://charts.loft.sh \ +// --namespace -system \ +// --repository-config='' \ +// --create-namespace + +// Need also +// syncer: +// extraArgs: +// - --tls-san=.localtest.me +// - --out-kube-config-server=https://.localtest.me:9443 + +const ( + URL = "https://charts.loft.sh" + RepoName = "loft" + ChartName = "vcluster" + ReleaseName = "vcluster" +) + +var ( + configs = []string{ + "vcluster.image=rancher/k3s:v1.27.2-k3s1", + } +) + +func (r *VClusterReconciler) ReconcileChart(ctx context.Context, hcp *tenancyv1alpha1.ControlPlane) error { + localDNSName := util.GenerateDevLocalDNSName(hcp.Name) + configs = append(configs, fmt.Sprintf("syncer.extraArgs[0]=--tls-san=%s", localDNSName)) + configs = append(configs, fmt.Sprintf("syncer.extraArgs[1]=--out-kube-config-server=https://%s:9443", localDNSName)) + h := &helm.HelmHandler{ + URL: URL, + RepoName: RepoName, + ChartName: ChartName, + Namespace: util.GenerateNamespaceFromControlPlaneName(hcp.Name), + ReleaseName: ReleaseName, + Args: map[string]string{"set": strings.Join(configs, ",")}, + } + err := helm.Init(ctx, h) + if err != nil { + return err + } + + if !h.IsDeployed() { + err := h.Install() + if err != nil { + return err + } + } + return nil +} diff --git a/pkg/reconcilers/vcluster/reconciler.go b/pkg/reconcilers/vcluster/reconciler.go new file mode 100644 index 0000000..0dbef92 --- /dev/null +++ b/pkg/reconcilers/vcluster/reconciler.go @@ -0,0 +1,66 @@ +/* +Copyright 2023 The KubeStellar Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vcluster + +import ( + "context" + + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + clog "sigs.k8s.io/controller-runtime/pkg/log" + + tenancyv1alpha1 "github.com/kubestellar/kubeflex/api/v1alpha1" + "github.com/kubestellar/kubeflex/pkg/reconcilers/shared" +) + +const ( + ServiceName = "vcluster" + ServicePort = 443 +) + +// VClusterReconciler reconciles a OCM ControlPlane +type VClusterReconciler struct { + *shared.BaseReconciler +} + +func New(cl client.Client, scheme *runtime.Scheme) *VClusterReconciler { + return &VClusterReconciler{ + BaseReconciler: &shared.BaseReconciler{ + Client: cl, + Scheme: scheme, + }, + } +} + +func (r *VClusterReconciler) Reconcile(ctx context.Context, hcp *tenancyv1alpha1.ControlPlane) (ctrl.Result, error) { + _ = clog.FromContext(ctx) + + if err := r.BaseReconciler.ReconcileNamespace(ctx, hcp); err != nil { + return r.UpdateStatusForSyncingError(hcp, err) + } + + if err := r.ReconcileChart(ctx, hcp); err != nil { + return r.UpdateStatusForSyncingError(hcp, err) + } + + if err := r.ReconcileAPIServerIngress(ctx, hcp, ServiceName, ServicePort); err != nil { + return r.UpdateStatusForSyncingError(hcp, err) + } + + return r.UpdateStatusForSyncingSuccess(ctx, hcp) +} diff --git a/pkg/util/util.go b/pkg/util/util.go index 656a6fe..181efb6 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -25,15 +25,19 @@ import ( ) const ( - APIServerDeploymentName = "kube-apiserver" - OCMServerDeploymentName = "multicluster-controlplane" - CMDeploymentName = "kube-controller-manager" - ProjectName = "kubeflex" - DBReleaseName = "postgres" - SystemNamespace = "kubeflex-system" - IngressSecurePort = "9443" - AdminConfSecret = "admin-kubeconfig" - OCMKubeConfigSecret = "multicluster-controlplane-kubeconfig" + APIServerDeploymentName = "kube-apiserver" + OCMServerDeploymentName = "multicluster-controlplane" + VClusterServerDeploymentName = "vcluster" + CMDeploymentName = "kube-controller-manager" + ProjectName = "kubeflex" + DBReleaseName = "postgres" + SystemNamespace = "kubeflex-system" + IngressSecurePort = "9443" + AdminConfSecret = "admin-kubeconfig" + OCMKubeConfigSecret = "multicluster-controlplane-kubeconfig" + VClusterKubeConfigSecret = "vc-vcluster" + KubeconfigSecretKeyDefault = "kubeconfig" + KubeconfigSecretKeyVCluster = "config" ) func GenerateNamespaceFromControlPlaneName(name string) string { @@ -80,18 +84,34 @@ func GetKubeconfSecretNameByControlPlaneType(controlPlaneType string) string { return AdminConfSecret case string(tenancyv1alpha1.ControlPlaneTypeOCM): return OCMKubeConfigSecret + case string(tenancyv1alpha1.ControlPlaneTypeVCluster): + return VClusterKubeConfigSecret default: // TODO - should we instead throw an error? return AdminConfSecret } } +func GetKubeconfSecretKeyNameByControlPlaneType(controlPlaneType string) string { + switch controlPlaneType { + case string(tenancyv1alpha1.ControlPlaneTypeK8S), string(tenancyv1alpha1.ControlPlaneTypeOCM): + return KubeconfigSecretKeyDefault + case string(tenancyv1alpha1.ControlPlaneTypeVCluster): + return KubeconfigSecretKeyVCluster + default: + // TODO - should we instead throw an error? + return KubeconfigSecretKeyDefault + } +} + func GetAPIServerDeploymentNameByControlPlaneType(controlPlaneType string) string { switch controlPlaneType { case string(tenancyv1alpha1.ControlPlaneTypeK8S): return APIServerDeploymentName case string(tenancyv1alpha1.ControlPlaneTypeOCM): return OCMServerDeploymentName + case string(tenancyv1alpha1.ControlPlaneTypeVCluster): + return VClusterServerDeploymentName default: // TODO - should we instead throw an error? return APIServerDeploymentName