Merge pull request #58 from pdettori/vcluster

✨ add support for vcluster control planes

cmd/kflex/create/create.go

@@ -59,12 +59,22 @@ func (c *CPCreate) Create(controlPlaneType, backendType string) {
 	util.PrintStatus("Waiting for API server to become ready...", done, &wg)
 	kubeconfig.WatchForSecretCreation(clientset, c.Name, util.GetKubeconfSecretNameByControlPlaneType(controlPlaneType))
 
-	if err := util.WaitForDeploymentReady(clientset,
-		util.GetAPIServerDeploymentNameByControlPlaneType(controlPlaneType),
-		util.GenerateNamespaceFromControlPlaneName(cp.Name)); err != nil {
-
-		fmt.Fprintf(os.Stderr, "Error waiting for deployment to become ready: %v\n", err)
-		os.Exit(1)
+	if controlPlaneType == string(tenancyv1alpha1.ControlPlaneTypeVCluster) {
+		if err := util.WaitForStatefulSetReady(clientset,
+			util.GetAPIServerDeploymentNameByControlPlaneType(controlPlaneType),
+			util.GenerateNamespaceFromControlPlaneName(cp.Name)); err != nil {
+
+			fmt.Fprintf(os.Stderr, "Error waiting for stateful set to become ready: %v\n", err)
+			os.Exit(1)
+		}
+	} else {
+		if err := util.WaitForDeploymentReady(clientset,
+			util.GetAPIServerDeploymentNameByControlPlaneType(controlPlaneType),
+			util.GenerateNamespaceFromControlPlaneName(cp.Name)); err != nil {
+
+			fmt.Fprintf(os.Stderr, "Error waiting for deployment to become ready: %v\n", err)
+			os.Exit(1)
+		}
 	}
 	done <- true
 

internal/controller/controlplane_controller.go

@@ -32,6 +32,7 @@ import (
 	tenancyv1alpha1 "github.com/kubestellar/kubeflex/api/v1alpha1"
 	"github.com/kubestellar/kubeflex/pkg/reconcilers/k8s"
 	"github.com/kubestellar/kubeflex/pkg/reconcilers/ocm"
+	"github.com/kubestellar/kubeflex/pkg/reconcilers/vcluster"
 	"github.com/kubestellar/kubeflex/pkg/util"
 )
 
@@ -94,6 +95,9 @@ func (r *ControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Request
 	case tenancyv1alpha1.ControlPlaneTypeOCM:
 		reconciler := ocm.New(r.Client, r.Scheme)
 		return reconciler.Reconcile(ctx, hcp)
+	case tenancyv1alpha1.ControlPlaneTypeVCluster:
+		reconciler := vcluster.New(r.Client, r.Scheme)
+		return reconciler.Reconcile(ctx, hcp)
 	default:
 		return ctrl.Result{}, fmt.Errorf("unsupported control plane type: %s", hcp.Spec.Type)
 	}

pkg/certs/kconfig_gen.go

@@ -44,7 +44,6 @@ const (
 	Organization     = "system:masters"
 	ContrCMCN        = "system:kube-controller-manager"
 	CMConfSecret     = "cm-kubeconfig"
-	ConfSecretKey    = "kubeconfig"
 )
 
 type ConfigGen struct {
@@ -85,7 +84,7 @@ func (c *ConfigGen) genSecretManifest(ctx context.Context, conf []byte) *v1.Secr
 		},
 		Type: v1.SecretTypeOpaque,
 		Data: map[string][]byte{
-			ConfSecretKey: conf,
+			util.KubeconfigSecretKeyDefault: conf,
 		},
 	}
 }

pkg/kubeconfig/kubeconfig.go

@@ -62,7 +62,8 @@ func loadControlPlaneKubeconfig(ctx context.Context, client kubernetes.Clientset
 		return nil, err
 	}
 
-	return clientcmd.Load(ks.Data[certs.ConfSecretKey])
+	key := util.GetKubeconfSecretKeyNameByControlPlaneType(controlPlaneType)
+	return clientcmd.Load(ks.Data[key])
 }
 
 func LoadKubeconfig(ctx context.Context) (*clientcmdapi.Config, error) {
@@ -117,6 +118,15 @@ func adjustConfigKeys(config *clientcmdapi.Config, cpName, controlPlaneType stri
 			Cluster:  certs.GenerateClusterName(cpName),
 			AuthInfo: certs.GenerateAuthInfoAdminName(cpName),
 		}
+	case string(tenancyv1alpha1.ControlPlaneTypeVCluster):
+		renameKey(config.Clusters, "my-vcluster", certs.GenerateClusterName(cpName))
+		renameKey(config.AuthInfos, "my-vcluster", certs.GenerateAuthInfoAdminName(cpName))
+		renameKey(config.Contexts, "my-vcluster", certs.GenerateContextName(cpName))
+		config.CurrentContext = certs.GenerateContextName(cpName)
+		config.Contexts[certs.GenerateContextName(cpName)] = &clientcmdapi.Context{
+			Cluster:  certs.GenerateClusterName(cpName),
+			AuthInfo: certs.GenerateAuthInfoAdminName(cpName),
+		}
 	default:
 		return
 	}

pkg/reconcilers/k8s/reconciler.go

@@ -77,7 +77,7 @@ func (r *K8sReconciler) Reconcile(ctx context.Context, hcp *tenancyv1alpha1.Cont
 		return r.UpdateStatusForSyncingError(hcp, err)
 	}
 
-	if err = r.ReconcileAPIServerIngress(ctx, hcp, ""); err != nil {
+	if err = r.ReconcileAPIServerIngress(ctx, hcp, "", shared.SecurePort); err != nil {
 		return r.UpdateStatusForSyncingError(hcp, err)
 	}
 

pkg/reconcilers/ocm/chart.go

@@ -26,18 +26,6 @@ import (
 	"github.com/kubestellar/kubeflex/pkg/util"
 )
 
-// helm install -n cp3-system multicluster-controlplane \
-//    oci://quay.io/pdettori/multicluster-controlplane-chart \
-//   --version 0.1.0 \
-//   --create-namespace \
-//   --set image=quay.io/pdettori/multicluster-controlplane:latest \
-//   --set route.enabled=false \
-//   --set apiserver.externalHostname=cp3.localtest.me \
-//   --set apiserver.internalHostname=kubeflex-control-plane \
-//   --set apiserver.port=9443 \
-//   --set nodeport.enabled=true \
-//   --set nodeport.port=30443
-
 const (
 	URL         = "oci://quay.io/pdettori/multicluster-controlplane-chart:0.1.0"
 	RepoName    = "multicluster-controlplane"
@@ -53,10 +41,6 @@ var (
 		"apiserver.port=9443",
 		"nodeport.enabled=false",
 	}
-
-	Args = map[string]string{
-		"set": "image=quay.io/pdettori/multicluster-controlplane:latest,route.enabled=false",
-	}
 )
 
 func (r *OCMReconciler) ReconcileChart(ctx context.Context, hcp *tenancyv1alpha1.ControlPlane) error {

pkg/reconcilers/ocm/reconciler.go

@@ -57,7 +57,7 @@ func (r *OCMReconciler) Reconcile(ctx context.Context, hcp *tenancyv1alpha1.Cont
 		return r.UpdateStatusForSyncingError(hcp, err)
 	}
 
-	if err := r.ReconcileAPIServerIngress(ctx, hcp, ServiceName); err != nil {
+	if err := r.ReconcileAPIServerIngress(ctx, hcp, ServiceName, shared.SecurePort); err != nil {
 		return r.UpdateStatusForSyncingError(hcp, err)
 	}
 

pkg/reconcilers/shared/ingress.go

@@ -39,7 +39,7 @@ var (
 	pathTypePrefix = networkingv1.PathTypePrefix
 )
 
-func (r *BaseReconciler) ReconcileAPIServerIngress(ctx context.Context, hcp *tenancyv1alpha1.ControlPlane, svcName string) error {
+func (r *BaseReconciler) ReconcileAPIServerIngress(ctx context.Context, hcp *tenancyv1alpha1.ControlPlane, svcName string, svcPort int) error {
 	_ = clog.FromContext(ctx)
 	namespace := util.GenerateNamespaceFromControlPlaneName(hcp.Name)
 
@@ -58,7 +58,7 @@ func (r *BaseReconciler) ReconcileAPIServerIngress(ctx context.Context, hcp *ten
 	err := r.Client.Get(context.TODO(), client.ObjectKeyFromObject(ingress), ingress, &client.GetOptions{})
 	if err != nil {
 		if apierrors.IsNotFound(err) {
-			ingress = generateAPIServerIngress(hcp.Name, svcName, namespace)
+			ingress = generateAPIServerIngress(hcp.Name, svcName, namespace, svcPort)
 			if err := controllerutil.SetControllerReference(hcp, ingress, r.Scheme); err != nil {
 				return nil
 			}
@@ -71,7 +71,7 @@ func (r *BaseReconciler) ReconcileAPIServerIngress(ctx context.Context, hcp *ten
 	return nil
 }
 
-func generateAPIServerIngress(name, svcName, namespace string) *networkingv1.Ingress {
+func generateAPIServerIngress(name, svcName, namespace string, svcPort int) *networkingv1.Ingress {
 	return &networkingv1.Ingress{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "Ingress",
@@ -99,7 +99,7 @@ func generateAPIServerIngress(name, svcName, namespace string) *networkingv1.Ing
 										Service: &networkingv1.IngressServiceBackend{
 											Name: svcName,
 											Port: networkingv1.ServiceBackendPort{
-												Number: SecurePort,
+												Number: int32(svcPort),
 											},
 										},
 									},

pkg/reconcilers/vcluster/chart.go

@@ -0,0 +1,79 @@
+/*
+Copyright 2023 The KubeStellar Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package vcluster
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	tenancyv1alpha1 "github.com/kubestellar/kubeflex/api/v1alpha1"
+	"github.com/kubestellar/kubeflex/pkg/helm"
+	"github.com/kubestellar/kubeflex/pkg/util"
+)
+
+// helm upgrade --install <controlplane-name> vcluster \
+//   --set vcluster.image=rancher/k3s:v1.27.2-k3s1 \
+//   --repo https://charts.loft.sh \
+//   --namespace <controlplane-name>-system \
+//   --repository-config='' \
+//   --create-namespace
+
+// Need also
+// syncer:
+//   extraArgs:
+//   - --tls-san=<controlplane-name>.localtest.me
+//   - --out-kube-config-server=https://<controlplane-name>.localtest.me:9443
+
+const (
+	URL         = "https://charts.loft.sh"
+	RepoName    = "loft"
+	ChartName   = "vcluster"
+	ReleaseName = "vcluster"
+)
+
+var (
+	configs = []string{
+		"vcluster.image=rancher/k3s:v1.27.2-k3s1",
+	}
+)
+
+func (r *VClusterReconciler) ReconcileChart(ctx context.Context, hcp *tenancyv1alpha1.ControlPlane) error {
+	localDNSName := util.GenerateDevLocalDNSName(hcp.Name)
+	configs = append(configs, fmt.Sprintf("syncer.extraArgs[0]=--tls-san=%s", localDNSName))
+	configs = append(configs, fmt.Sprintf("syncer.extraArgs[1]=--out-kube-config-server=https://%s:9443", localDNSName))
+	h := &helm.HelmHandler{
+		URL:         URL,
+		RepoName:    RepoName,
+		ChartName:   ChartName,
+		Namespace:   util.GenerateNamespaceFromControlPlaneName(hcp.Name),
+		ReleaseName: ReleaseName,
+		Args:        map[string]string{"set": strings.Join(configs, ",")},
+	}
+	err := helm.Init(ctx, h)
+	if err != nil {
+		return err
+	}
+
+	if !h.IsDeployed() {
+		err := h.Install()
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

pkg/reconcilers/vcluster/reconciler.go

@@ -0,0 +1,66 @@
+/*
+Copyright 2023 The KubeStellar Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package vcluster
+
+import (
+	"context"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	clog "sigs.k8s.io/controller-runtime/pkg/log"
+
+	tenancyv1alpha1 "github.com/kubestellar/kubeflex/api/v1alpha1"
+	"github.com/kubestellar/kubeflex/pkg/reconcilers/shared"
+)
+
+const (
+	ServiceName = "vcluster"
+	ServicePort = 443
+)
+
+// VClusterReconciler reconciles a OCM ControlPlane
+type VClusterReconciler struct {
+	*shared.BaseReconciler
+}
+
+func New(cl client.Client, scheme *runtime.Scheme) *VClusterReconciler {
+	return &VClusterReconciler{
+		BaseReconciler: &shared.BaseReconciler{
+			Client: cl,
+			Scheme: scheme,
+		},
+	}
+}
+
+func (r *VClusterReconciler) Reconcile(ctx context.Context, hcp *tenancyv1alpha1.ControlPlane) (ctrl.Result, error) {
+	_ = clog.FromContext(ctx)
+
+	if err := r.BaseReconciler.ReconcileNamespace(ctx, hcp); err != nil {
+		return r.UpdateStatusForSyncingError(hcp, err)
+	}
+
+	if err := r.ReconcileChart(ctx, hcp); err != nil {
+		return r.UpdateStatusForSyncingError(hcp, err)
+	}
+
+	if err := r.ReconcileAPIServerIngress(ctx, hcp, ServiceName, ServicePort); err != nil {
+		return r.UpdateStatusForSyncingError(hcp, err)
+	}
+
+	return r.UpdateStatusForSyncingSuccess(ctx, hcp)
+}

pkg/util/util.go

@@ -25,15 +25,19 @@ import (
 )
 
 const (
-	APIServerDeploymentName = "kube-apiserver"
-	OCMServerDeploymentName = "multicluster-controlplane"
-	CMDeploymentName        = "kube-controller-manager"
-	ProjectName             = "kubeflex"
-	DBReleaseName           = "postgres"
-	SystemNamespace         = "kubeflex-system"
-	IngressSecurePort       = "9443"
-	AdminConfSecret         = "admin-kubeconfig"
-	OCMKubeConfigSecret     = "multicluster-controlplane-kubeconfig"
+	APIServerDeploymentName      = "kube-apiserver"
+	OCMServerDeploymentName      = "multicluster-controlplane"
+	VClusterServerDeploymentName = "vcluster"
+	CMDeploymentName             = "kube-controller-manager"
+	ProjectName                  = "kubeflex"
+	DBReleaseName                = "postgres"
+	SystemNamespace              = "kubeflex-system"
+	IngressSecurePort            = "9443"
+	AdminConfSecret              = "admin-kubeconfig"
+	OCMKubeConfigSecret          = "multicluster-controlplane-kubeconfig"
+	VClusterKubeConfigSecret     = "vc-vcluster"
+	KubeconfigSecretKeyDefault   = "kubeconfig"
+	KubeconfigSecretKeyVCluster  = "config"
 )
 
 func GenerateNamespaceFromControlPlaneName(name string) string {
@@ -80,18 +84,34 @@ func GetKubeconfSecretNameByControlPlaneType(controlPlaneType string) string {
 		return AdminConfSecret
 	case string(tenancyv1alpha1.ControlPlaneTypeOCM):
 		return OCMKubeConfigSecret
+	case string(tenancyv1alpha1.ControlPlaneTypeVCluster):
+		return VClusterKubeConfigSecret
 	default:
 		// TODO - should we instead throw an error?
 		return AdminConfSecret
 	}
 }
 
+func GetKubeconfSecretKeyNameByControlPlaneType(controlPlaneType string) string {
+	switch controlPlaneType {
+	case string(tenancyv1alpha1.ControlPlaneTypeK8S), string(tenancyv1alpha1.ControlPlaneTypeOCM):
+		return KubeconfigSecretKeyDefault
+	case string(tenancyv1alpha1.ControlPlaneTypeVCluster):
+		return KubeconfigSecretKeyVCluster
+	default:
+		// TODO - should we instead throw an error?
+		return KubeconfigSecretKeyDefault
+	}
+}
+
 func GetAPIServerDeploymentNameByControlPlaneType(controlPlaneType string) string {
 	switch controlPlaneType {
 	case string(tenancyv1alpha1.ControlPlaneTypeK8S):
 		return APIServerDeploymentName
 	case string(tenancyv1alpha1.ControlPlaneTypeOCM):
 		return OCMServerDeploymentName
+	case string(tenancyv1alpha1.ControlPlaneTypeVCluster):
+		return VClusterServerDeploymentName
 	default:
 		// TODO - should we instead throw an error?
 		return APIServerDeploymentName