feat: install api-server-external-service in virtualcluster

Signed-off-by: qiuwei <[email protected]>

pkg/kubenest/constants/constant.go

@@ -9,6 +9,7 @@ const (
 	KosmosJoinControllerName      = "kosmos-join-controller"
 	KosmosNs                      = "kosmos-system"
 	SystemNs                      = "kube-system"
+	DefaultNs                     = "default"
 	DefaultImageRepositoryEnv     = "IMAGE_REPOSITIRY"
 	DefaultImageVersionEnv        = "IMAGE_VERSION"
 	DefaultCoreDnsImageTagEnv     = "COREDNS_IMAGE_TAG"
@@ -112,6 +113,9 @@ const (
 	StateLabelKey = "kosmos-io/state"
 
 	KonnectivityServerSuffix = "konnectivity-server"
+
+	//in virtual cluster
+	ApiServerExternalService = "api-server-external-service"
 )
 
 type Action string

pkg/kubenest/controlplane/endpoint.go

@@ -0,0 +1,158 @@
+package controlplane
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/pkg/errors"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/yaml"
+	"k8s.io/client-go/dynamic"
+	"k8s.io/klog/v2"
+
+	"github.com/kosmos.io/kosmos/pkg/kubenest/constants"
+	"github.com/kosmos.io/kosmos/pkg/kubenest/manifest/controlplane/virtualcluster"
+	"github.com/kosmos.io/kosmos/pkg/kubenest/util"
+)
+
+func EnsureApiServerExternalEndPoint(dynamicClient dynamic.Interface) error {
+	err := installApiServerExternalEndpointInVirtualCluster(dynamicClient)
+	if err != nil {
+		return err
+	}
+
+	err = installApiServerExternalServiceInVirtualCluster(dynamicClient)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func installApiServerExternalEndpointInVirtualCluster(dynamicClient dynamic.Interface) error {
+	klog.Info("begin to get kubernetes endpoint")
+	kubeEndpointUnstructured, err := dynamicClient.Resource(schema.GroupVersionResource{
+		Group:    "",
+		Version:  "v1",
+		Resource: "endpoints",
+	}).Namespace(constants.DefaultNs).Get(context.TODO(), "kubernetes", metav1.GetOptions{})
+	if err != nil {
+		klog.Error("get Kubernetes endpoint failed", err)
+		return errors.Wrap(err, "failed to get kubernetes endpoint")
+	}
+	klog.V(4).Info("the Kubernetes endpoint is：", kubeEndpointUnstructured)
+
+	if kubeEndpointUnstructured != nil {
+		kubeEndpoint := &corev1.Endpoints{}
+		err := runtime.DefaultUnstructuredConverter.FromUnstructured(kubeEndpointUnstructured.Object, kubeEndpoint)
+		if err != nil {
+			klog.Error("switch Kubernetes endpoint to typed object failed", err)
+			return errors.Wrap(err, "failed to convert kubernetes endpoint to typed object")
+		}
+
+		newEndpoint := kubeEndpoint.DeepCopy()
+		newEndpoint.Name = constants.ApiServerExternalService
+		newEndpoint.Namespace = constants.DefaultNs
+		newEndpoint.ResourceVersion = ""
+		newEndpointUnstructuredObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(newEndpoint)
+		if err != nil {
+			klog.Error("switch new endpoint to unstructured object failed", err)
+			return errors.Wrap(err, "failed to convert new endpoint to unstructured object")
+		}
+		klog.V(4).Info("after switch the Endpoint unstructured is：", newEndpointUnstructuredObj)
+
+		newEndpointUnstructured := &unstructured.Unstructured{Object: newEndpointUnstructuredObj}
+		createResult, err := dynamicClient.Resource(schema.GroupVersionResource{
+			Group:    "",
+			Version:  "v1",
+			Resource: "endpoints",
+		}).Namespace(constants.DefaultNs).Create(context.TODO(), newEndpointUnstructured, metav1.CreateOptions{})
+		if err != nil {
+			klog.Error("create api-server-external-service endpoint failed", err)
+			return errors.Wrap(err, "failed to create api-server-external-service endpoint")
+		} else {
+			klog.Info("success create api-server-external-service endpoint:", createResult)
+		}
+	} else {
+		return errors.New("kubernetes endpoint does not exist")
+	}
+
+	return nil
+}
+
+func installApiServerExternalServiceInVirtualCluster(dynamicClient dynamic.Interface) error {
+	port, err := getEndPointPort(dynamicClient)
+	if err != nil {
+		return fmt.Errorf("error when getEndPointPort: %w", err)
+	}
+	apiServerExternalServiceBytes, err := util.ParseTemplate(virtualcluster.ApiServerExternalService, struct {
+		ServicePort int32
+	}{
+		ServicePort: port,
+	})
+	if err != nil {
+		return fmt.Errorf("error when parsing api-server-external-serive template: %w", err)
+	}
+
+	var obj unstructured.Unstructured
+	if err := yaml.Unmarshal([]byte(apiServerExternalServiceBytes), &obj); err != nil {
+		return fmt.Errorf("err when decoding api-server-external service in virtual cluster: %w", err)
+	}
+
+	err = util.CreateObject(dynamicClient, "default", "api-server-external-service", &obj)
+	if err != nil {
+		return fmt.Errorf("error when creating api-server-external service in virtual cluster err: %w", err)
+	}
+	return nil
+}
+
+func getEndPointPort(dynamicClient dynamic.Interface) (int32, error) {
+	klog.Info("begin to get Endpoints ports...")
+	endpointsRes := dynamicClient.Resource(schema.GroupVersionResource{
+		Group:    "",
+		Version:  "v1",
+		Resource: "endpoints",
+	}).Namespace(constants.DefaultNs)
+
+	endpointsRaw, err := endpointsRes.Get(context.TODO(), constants.ApiServerExternalService, metav1.GetOptions{})
+	if err != nil {
+		klog.Errorf("get Endpoints failed: %v", err)
+		return 0, err
+	}
+
+	subsets, found, err := unstructured.NestedSlice(endpointsRaw.Object, "subsets")
+	if !found || err != nil {
+		klog.Errorf("The subsets field was not found or parsing error occurred: %v", err)
+		return 0, fmt.Errorf("subsets field not found or error parsing it")
+	}
+
+	if len(subsets) == 0 {
+		klog.Errorf("subsets is empty")
+		return 0, fmt.Errorf("No subsets found in the endpoints")
+	}
+
+	subset := subsets[0].(map[string]interface{})
+	ports, found, err := unstructured.NestedSlice(subset, "ports")
+	if !found || err != nil {
+		klog.Errorf("ports field not found or parsing error: %v", err)
+		return 0, fmt.Errorf("ports field not found or error parsing it")
+	}
+
+	if len(ports) == 0 {
+		klog.Errorf("Port not found in the endpoint")
+		return 0, fmt.Errorf("No ports found in the endpoint")
+	}
+
+	port := ports[0].(map[string]interface{})
+	portNum, found, err := unstructured.NestedInt64(port, "port")
+	if !found || err != nil {
+		klog.Errorf("ports field not found or parsing error: %v", err)
+		return 0, fmt.Errorf("port field not found or error parsing it")
+	}
+
+	klog.Infof("The port number was successfully obtained: %d", portNum)
+	return int32(portNum), nil
+}

pkg/kubenest/init.go

@@ -68,6 +68,7 @@ func NewInitPhase(opts *InitOptions) *workflow.Phase {
 	initPhase.AppendTask(tasks.NewCoreDNSTask())
 	// add server
 	initPhase.AppendTask(tasks.NewComponentsFromManifestsTask())
+	initPhase.AppendTask(tasks.NewEndPointTask())
 
 	initPhase.SetDataInitializer(func() (workflow.RunData, error) {
 		return newRunData(opts)

pkg/kubenest/manifest/controlplane/virtualcluster/manifests_service.go

@@ -0,0 +1,18 @@
+package virtualcluster
+
+const (
+	ApiServerExternalService = `
+apiVersion: v1
+kind: Service
+metadata:
+  name: api-server-external-service
+  namespace: default
+spec:
+  type: NodePort
+  ports:
+    - protocol: TCP
+      port: {{ .ServicePort }}
+      targetPort: {{ .ServicePort }}
+      nodePort: 30443
+`
+)

pkg/kubenest/tasks/coredns.go

@@ -133,7 +133,7 @@ func runCheckCoreDnsTask(r workflow.RunData) error {
 func runCoreDnsVirtualTask(r workflow.RunData) error {
 	data, ok := r.(InitData)
 	if !ok {
-		return errors.New("Virtual cluster manifests-components task invoked with an invalid data struct")
+		return errors.New("Virtual cluster coreDns task invoked with an invalid data struct")
 	}
 
 	secret, err := data.RemoteClient().CoreV1().Secrets(data.GetNamespace()).Get(context.TODO(),

pkg/kubenest/tasks/endpoint.go

@@ -0,0 +1,67 @@
+package tasks
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/pkg/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/dynamic"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/klog/v2"
+
+	"github.com/kosmos.io/kosmos/pkg/kubenest/constants"
+	"github.com/kosmos.io/kosmos/pkg/kubenest/controlplane"
+	"github.com/kosmos.io/kosmos/pkg/kubenest/workflow"
+)
+
+func NewEndPointTask() workflow.Task {
+	return workflow.Task{
+		Name:        "endpoint",
+		Run:         runEndpoint,
+		RunSubTasks: true,
+		Tasks: []workflow.Task{
+			{
+				Name: "deploy-endpoint-in-virtual-cluster",
+				Run:  runEndPointInVirtualClusterTask,
+			},
+		},
+	}
+}
+
+func runEndpoint(r workflow.RunData) error {
+	data, ok := r.(InitData)
+	if !ok {
+		return errors.New("endPoint task invoked with an invalid data struct")
+	}
+
+	klog.V(4).InfoS("[endPoint] Running endPoint task", "virtual cluster", klog.KObj(data))
+	return nil
+}
+
+func runEndPointInVirtualClusterTask(r workflow.RunData) error {
+	data, ok := r.(InitData)
+	if !ok {
+		return errors.New("Virtual cluster endpoint task invoked with an invalid data struct")
+	}
+
+	secret, err := data.RemoteClient().CoreV1().Secrets(data.GetNamespace()).Get(context.TODO(),
+		fmt.Sprintf("%s-%s", data.GetName(), constants.AdminConfig), metav1.GetOptions{})
+	if err != nil {
+		return errors.Wrap(err, "Get virtualcluster kubeconfig secret error")
+	}
+	config, err := clientcmd.RESTConfigFromKubeConfig(secret.Data[constants.KubeConfig])
+	if err != nil {
+		return err
+	}
+	dynamicClient, err := dynamic.NewForConfig(config)
+	if err != nil {
+		return err
+	}
+
+	err = controlplane.EnsureApiServerExternalEndPoint(dynamicClient)
+	if err != nil {
+		return err
+	}
+	return nil
+}