From 14ff9e69791453844a83929c776746a451dc8011 Mon Sep 17 00:00:00 2001 From: qiuwei Date: Tue, 11 Jun 2024 19:39:45 +0800 Subject: [PATCH] feat: install api-server-external-service in virtualcluster Signed-off-by: qiuwei --- pkg/kubenest/constants/constant.go | 4 + pkg/kubenest/controlplane/endpoint.go | 160 ++++++++++++++++++ pkg/kubenest/init.go | 1 + .../virtualcluster/manifests_service.go | 18 ++ pkg/kubenest/tasks/coredns.go | 2 +- pkg/kubenest/tasks/endpoint.go | 67 ++++++++ 6 files changed, 251 insertions(+), 1 deletion(-) create mode 100644 pkg/kubenest/controlplane/endpoint.go create mode 100644 pkg/kubenest/manifest/controlplane/virtualcluster/manifests_service.go create mode 100644 pkg/kubenest/tasks/endpoint.go diff --git a/pkg/kubenest/constants/constant.go b/pkg/kubenest/constants/constant.go index ab9eab563..52f853ddb 100644 --- a/pkg/kubenest/constants/constant.go +++ b/pkg/kubenest/constants/constant.go @@ -9,6 +9,7 @@ const ( KosmosJoinControllerName = "kosmos-join-controller" KosmosNs = "kosmos-system" SystemNs = "kube-system" + DefaultNs = "default" DefaultImageRepositoryEnv = "IMAGE_REPOSITIRY" DefaultImageVersionEnv = "IMAGE_VERSION" DefaultCoreDnsImageTagEnv = "COREDNS_IMAGE_TAG" @@ -112,6 +113,9 @@ const ( StateLabelKey = "kosmos-io/state" KonnectivityServerSuffix = "konnectivity-server" + + //in virtual cluster + ApiServerExternalService = "api-server-external-service" ) type Action string diff --git a/pkg/kubenest/controlplane/endpoint.go b/pkg/kubenest/controlplane/endpoint.go new file mode 100644 index 000000000..e1795bc90 --- /dev/null +++ b/pkg/kubenest/controlplane/endpoint.go @@ -0,0 +1,160 @@ +package controlplane + +import ( + "context" + "fmt" + + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/client-go/dynamic" + "k8s.io/klog/v2" + + "github.com/kosmos.io/kosmos/pkg/kubenest/constants" + "github.com/kosmos.io/kosmos/pkg/kubenest/manifest/controlplane/virtualcluster" + "github.com/kosmos.io/kosmos/pkg/kubenest/util" +) + +func EnsureApiServerExternalEndPoint(dynamicClient dynamic.Interface) error { + err := installApiServerExternalEndpointInVirtualCluster(dynamicClient) + if err != nil { + return err + } + + err = installApiServerExternalServiceInVirtualCluster(dynamicClient) + if err != nil { + return err + } + return nil +} + +func installApiServerExternalEndpointInVirtualCluster(dynamicClient dynamic.Interface) error { + klog.Info("begin to get kubernetes endpoint") + kubeEndpointUnstructured, err := dynamicClient.Resource(schema.GroupVersionResource{ + Group: "", + Version: "v1", + Resource: "endpoints", + }).Namespace(constants.DefaultNs).Get(context.TODO(), "kubernetes", metav1.GetOptions{}) + if err != nil { + klog.Error("get Kubernetes endpoint failed", err) + return errors.Wrap(err, "failed to get kubernetes endpoint") + } + klog.V(4).Info("the Kubernetes endpoint is：", kubeEndpointUnstructured) + + if kubeEndpointUnstructured != nil { + kubeEndpoint := &corev1.Endpoints{} + err := runtime.DefaultUnstructuredConverter.FromUnstructured(kubeEndpointUnstructured.Object, kubeEndpoint) + if err != nil { + klog.Error("switch Kubernetes endpoint to typed object failed", err) + return errors.Wrap(err, "failed to convert kubernetes endpoint to typed object") + } + + newEndpoint := kubeEndpoint.DeepCopy() + newEndpoint.Name = constants.ApiServerExternalService + newEndpoint.Namespace = constants.DefaultNs + newEndpoint.ResourceVersion = "" + newEndpointUnstructuredObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(newEndpoint) + if err != nil { + klog.Error("switch new endpoint to unstructured object failed", err) + return errors.Wrap(err, "failed to convert new endpoint to unstructured object") + } + klog.V(4).Info("after switch the Endpoint unstructured is：", newEndpointUnstructuredObj) + + newEndpointUnstructured := &unstructured.Unstructured{Object: newEndpointUnstructuredObj} + createResult, err := dynamicClient.Resource(schema.GroupVersionResource{ + Group: "", + Version: "v1", + Resource: "endpoints", + }).Namespace(constants.DefaultNs).Create(context.TODO(), newEndpointUnstructured, metav1.CreateOptions{}) + if err != nil { + klog.Error("create api-server-external-service endpoint failed", err) + return errors.Wrap(err, "failed to create api-server-external-service endpoint") + } else { + klog.Info("success create api-server-external-service endpoint:", createResult) + } + } else { + return errors.New("kubernetes endpoint does not exist") + } + + return nil +} + +func installApiServerExternalServiceInVirtualCluster(dynamicClient dynamic.Interface) error { + port, err := getEndPointPort(dynamicClient) + if err != nil { + return fmt.Errorf("error when getEndPointPort: %w", err) + } + apiServerExternalServiceBytes, err := util.ParseTemplate(virtualcluster.ApiServerExternalService, struct { + ServicePort int32 + }{ + ServicePort: port, + }) + if err != nil { + return fmt.Errorf("error when parsing api-server-external-serive template: %w", err) + } + + var obj unstructured.Unstructured + if err := yaml.Unmarshal([]byte(apiServerExternalServiceBytes), &obj); err != nil { + return fmt.Errorf("err when decoding api-server-external service in virtual cluster: %w", err) + } + + err = util.CreateObject(dynamicClient, "default", "api-server-external-service", &obj) + if err != nil { + return fmt.Errorf("error when creating api-server-external service in virtual cluster err: %w", err) + } + return nil + +} + +func getEndPointPort(dynamicClient dynamic.Interface) (int32, error) { + klog.Info("begin to get Endpoints ports...") + endpointsRes := dynamicClient.Resource(schema.GroupVersionResource{ + Group: "", + Version: "v1", + Resource: "endpoints", + }).Namespace(constants.DefaultNs) + + endpointsRaw, err := endpointsRes.Get(context.TODO(), constants.ApiServerExternalService, metav1.GetOptions{}) + if err != nil { + klog.Errorf("get Endpoints failed: %v", err) + return 0, err + } + + subsets, found, err := unstructured.NestedSlice(endpointsRaw.Object, "subsets") + if !found || err != nil { + klog.Errorf("The subsets field was not found or parsing error occurred: %v", err) + return 0, fmt.Errorf("subsets field not found or error parsing it") + } + + if len(subsets) == 0 { + klog.Errorf("subsets is empty") + return 0, fmt.Errorf("No subsets found in the endpoints") + } + + subset := subsets[0].(map[string]interface{}) + ports, found, err := unstructured.NestedSlice(subset, "ports") + if !found || err != nil { + klog.Errorf("ports field not found or parsing error: %v", err) + return 0, fmt.Errorf("ports field not found or error parsing it") + } + + if len(ports) == 0 { + klog.Errorf("Port not found in the endpoint") + return 0, fmt.Errorf("No ports found in the endpoint") + } + + port := ports[0].(map[string]interface{}) + portNum, found, err := unstructured.NestedInt64(port, "port") + if !found || err != nil { + klog.Errorf("ports field not found or parsing error: %v", err) + return 0, fmt.Errorf("port field not found or error parsing it") + } + + klog.Infof("The port number was successfully obtained: %d", portNum) + return int32(portNum), nil + +} diff --git a/pkg/kubenest/init.go b/pkg/kubenest/init.go index 2cecd402b..b36cd7ea3 100644 --- a/pkg/kubenest/init.go +++ b/pkg/kubenest/init.go @@ -68,6 +68,7 @@ func NewInitPhase(opts *InitOptions) *workflow.Phase { initPhase.AppendTask(tasks.NewCoreDNSTask()) // add server initPhase.AppendTask(tasks.NewComponentsFromManifestsTask()) + initPhase.AppendTask(tasks.NewEndPointTask()) initPhase.SetDataInitializer(func() (workflow.RunData, error) { return newRunData(opts) diff --git a/pkg/kubenest/manifest/controlplane/virtualcluster/manifests_service.go b/pkg/kubenest/manifest/controlplane/virtualcluster/manifests_service.go new file mode 100644 index 000000000..1ac24b81d --- /dev/null +++ b/pkg/kubenest/manifest/controlplane/virtualcluster/manifests_service.go @@ -0,0 +1,18 @@ +package virtualcluster + +const ( + ApiServerExternalService = ` +apiVersion: v1 +kind: Service +metadata: + name: api-server-external-service + namespace: default +spec: + type: NodePort + ports: + - protocol: TCP + port: {{ .ServicePort }} + targetPort: {{ .ServicePort }} + nodePort: 30443 +` +) diff --git a/pkg/kubenest/tasks/coredns.go b/pkg/kubenest/tasks/coredns.go index d4f2cf6e1..93562129e 100644 --- a/pkg/kubenest/tasks/coredns.go +++ b/pkg/kubenest/tasks/coredns.go @@ -133,7 +133,7 @@ func runCheckCoreDnsTask(r workflow.RunData) error { func runCoreDnsVirtualTask(r workflow.RunData) error { data, ok := r.(InitData) if !ok { - return errors.New("Virtual cluster manifests-components task invoked with an invalid data struct") + return errors.New("Virtual cluster coreDns task invoked with an invalid data struct") } secret, err := data.RemoteClient().CoreV1().Secrets(data.GetNamespace()).Get(context.TODO(), diff --git a/pkg/kubenest/tasks/endpoint.go b/pkg/kubenest/tasks/endpoint.go new file mode 100644 index 000000000..edfa81e6f --- /dev/null +++ b/pkg/kubenest/tasks/endpoint.go @@ -0,0 +1,67 @@ +package tasks + +import ( + "context" + "fmt" + + "github.com/pkg/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/klog/v2" + + "github.com/kosmos.io/kosmos/pkg/kubenest/constants" + "github.com/kosmos.io/kosmos/pkg/kubenest/controlplane" + "github.com/kosmos.io/kosmos/pkg/kubenest/workflow" +) + +func NewEndPointTask() workflow.Task { + return workflow.Task{ + Name: "endpoint", + Run: runEndpoint, + RunSubTasks: true, + Tasks: []workflow.Task{ + { + Name: "deploy-endpoint-in-virtual-cluster", + Run: runEndPointInVirtualClusterTask, + }, + }, + } +} + +func runEndpoint(r workflow.RunData) error { + data, ok := r.(InitData) + if !ok { + return errors.New("endPoint task invoked with an invalid data struct") + } + + klog.V(4).InfoS("[endPoint] Running endPoint task", "virtual cluster", klog.KObj(data)) + return nil +} + +func runEndPointInVirtualClusterTask(r workflow.RunData) error { + data, ok := r.(InitData) + if !ok { + return errors.New("Virtual cluster endpoint task invoked with an invalid data struct") + } + + secret, err := data.RemoteClient().CoreV1().Secrets(data.GetNamespace()).Get(context.TODO(), + fmt.Sprintf("%s-%s", data.GetName(), constants.AdminConfig), metav1.GetOptions{}) + if err != nil { + return errors.Wrap(err, "Get virtualcluster kubeconfig secret error") + } + config, err := clientcmd.RESTConfigFromKubeConfig(secret.Data[constants.KubeConfig]) + if err != nil { + return err + } + dynamicClient, err := dynamic.NewForConfig(config) + if err != nil { + return err + } + + err = controlplane.EnsureApiServerExternalEndPoint(dynamicClient) + if err != nil { + return err + } + return nil +}